<html><!-- Created using the cpp_pretty_printer from the dlib C++ library.  See http://dlib.net for updates. --><head><title>dlib C++ Library - dnn.cpp</title></head><body bgcolor='white'><pre>
<font color='#009900'>// Copyright (C) 2015  Davis E. King (davis@dlib.net)
</font><font color='#009900'>// License: Boost Software License   See LICENSE.txt for the full license.
</font>

<font color='#0000FF'>#include</font> <font color='#5555FF'>&lt;</font>sstream<font color='#5555FF'>&gt;</font>
<font color='#0000FF'>#include</font> <font color='#5555FF'>&lt;</font>string<font color='#5555FF'>&gt;</font>
<font color='#0000FF'>#include</font> <font color='#5555FF'>&lt;</font>cstdlib<font color='#5555FF'>&gt;</font>
<font color='#0000FF'>#include</font> <font color='#5555FF'>&lt;</font>ctime<font color='#5555FF'>&gt;</font>
<font color='#0000FF'>#include</font> <font color='#5555FF'>&lt;</font>vector<font color='#5555FF'>&gt;</font>
<font color='#0000FF'>#include</font> <font color='#5555FF'>&lt;</font>random<font color='#5555FF'>&gt;</font>
<font color='#0000FF'>#include</font> <font color='#5555FF'>&lt;</font>numeric<font color='#5555FF'>&gt;</font>
<font color='#0000FF'>#include</font> "<a style='text-decoration:none' href='../dnn.h.html'>../dnn.h</a>"

<font color='#0000FF'>#include</font> "<a style='text-decoration:none' href='tester.h.html'>tester.h</a>"

<font color='#0000FF'>#ifndef</font> __INTELLISENSE__

<font color='#0000FF'>namespace</font>
<b>{</b>

    <font color='#0000FF'>using</font> <font color='#0000FF'>namespace</font> test;
    <font color='#0000FF'>using</font> <font color='#0000FF'>namespace</font> dlib;
    <font color='#0000FF'>using</font> <font color='#0000FF'>namespace</font> std;

    logger <b><a name='dlog'></a>dlog</b><font face='Lucida Console'>(</font>"<font color='#CC0000'>test.dnn</font>"<font face='Lucida Console'>)</font>;

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> T<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'><u>float</u></font> <b><a name='compare_gradients'></a>compare_gradients</b> <font face='Lucida Console'>(</font>
        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> t,
        T grad
    <font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#0000FF'><u>float</u></font> max_error <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        <font color='#0000FF'>auto</font> p <font color='#5555FF'>=</font> t.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> t.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
        <b>{</b>
            max_error <font color='#5555FF'>=</font> std::<font color='#BB00BB'>max</font><font face='Lucida Console'>(</font>max_error, std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>p[i]<font color='#5555FF'>-</font><font color='#BB00BB'>grad</font><font face='Lucida Console'>(</font>i<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <b>}</b>
        <font color='#0000FF'>return</font> max_error;
    <b>}</b>

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'><u>void</u></font> <b><a name='test_tanh'></a>test_tanh</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#0000FF'>using</font> <font color='#0000FF'>namespace</font> dlib::tt;
        <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        resizable_tensor src, dest, gradient_input;
        src <font color='#5555FF'>=</font> matrix_cast<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font><font color='#BB00BB'>gaussian_randm</font><font face='Lucida Console'>(</font><font color='#979000'>5</font>,<font color='#979000'>5</font>, <font color='#979000'>0</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        dest <font color='#5555FF'>=</font> matrix_cast<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font><font color='#BB00BB'>gaussian_randm</font><font face='Lucida Console'>(</font><font color='#979000'>5</font>,<font color='#979000'>5</font>, <font color='#979000'>1</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        gradient_input <font color='#5555FF'>=</font> matrix_cast<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font><font color='#BB00BB'>gaussian_randm</font><font face='Lucida Console'>(</font><font color='#979000'>5</font>,<font color='#979000'>5</font>, <font color='#979000'>2</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;



        <font color='#0000FF'>auto</font> grad_src <font color='#5555FF'>=</font> [<font color='#5555FF'>&amp;</font>]<font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> idx<font face='Lucida Console'>)</font> <b>{</b>
            <font color='#0000FF'>auto</font> f <font color='#5555FF'>=</font> [<font color='#5555FF'>&amp;</font>]<font face='Lucida Console'>(</font><font color='#0000FF'><u>float</u></font> eps<font face='Lucida Console'>)</font> <b>{</b>
                <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> old <font color='#5555FF'>=</font> src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[idx];
                src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[idx] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> eps;
                <font color='#BB00BB'>tanh</font><font face='Lucida Console'>(</font>dest, src<font face='Lucida Console'>)</font>;
                <font color='#0000FF'><u>float</u></font> result <font color='#5555FF'>=</font> <font color='#BB00BB'>dot</font><font face='Lucida Console'>(</font>gradient_input, dest<font face='Lucida Console'>)</font>;
                src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[idx] <font color='#5555FF'>=</font> old;
                <font color='#0000FF'>return</font> result;
            <b>}</b>;
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> eps <font color='#5555FF'>=</font> <font color='#979000'>0.01</font>;
            <font color='#0000FF'>return</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>f</font><font face='Lucida Console'>(</font><font color='#5555FF'>+</font>eps<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>f</font><font face='Lucida Console'>(</font><font color='#5555FF'>-</font>eps<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font face='Lucida Console'>(</font><font color='#979000'>2</font><font color='#5555FF'>*</font>eps<font face='Lucida Console'>)</font>;
        <b>}</b>;

        resizable_tensor src_grad;
        src_grad.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;
        src_grad <font color='#5555FF'>=</font> <font color='#979000'>0</font>;

        <font color='#BB00BB'>tanh</font><font face='Lucida Console'>(</font>dest, src<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>tanh_gradient</font><font face='Lucida Console'>(</font>src_grad, dest, gradient_input<font face='Lucida Console'>)</font>;

        <font color='#0000FF'>auto</font> grad_error <font color='#5555FF'>=</font> <font color='#BB00BB'>compare_gradients</font><font face='Lucida Console'>(</font>src_grad, grad_src<font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>src error: </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> grad_error;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>grad_error <font color='#5555FF'>&lt;</font> <font color='#979000'>0.001</font><font face='Lucida Console'>)</font>;
    <b>}</b>

    <font color='#0000FF'><u>void</u></font> <b><a name='test_sigmoid'></a>test_sigmoid</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#0000FF'>using</font> <font color='#0000FF'>namespace</font> dlib::tt;
        <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        resizable_tensor src, dest, gradient_input;
        src <font color='#5555FF'>=</font> matrix_cast<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font><font color='#BB00BB'>gaussian_randm</font><font face='Lucida Console'>(</font><font color='#979000'>5</font>,<font color='#979000'>5</font>, <font color='#979000'>0</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        dest <font color='#5555FF'>=</font> matrix_cast<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font><font color='#BB00BB'>gaussian_randm</font><font face='Lucida Console'>(</font><font color='#979000'>5</font>,<font color='#979000'>5</font>, <font color='#979000'>1</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        gradient_input <font color='#5555FF'>=</font> matrix_cast<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font><font color='#BB00BB'>gaussian_randm</font><font face='Lucida Console'>(</font><font color='#979000'>5</font>,<font color='#979000'>5</font>, <font color='#979000'>2</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;



        <font color='#0000FF'>auto</font> grad_src <font color='#5555FF'>=</font> [<font color='#5555FF'>&amp;</font>]<font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> idx<font face='Lucida Console'>)</font> <b>{</b>
            <font color='#0000FF'>auto</font> f <font color='#5555FF'>=</font> [<font color='#5555FF'>&amp;</font>]<font face='Lucida Console'>(</font><font color='#0000FF'><u>float</u></font> eps<font face='Lucida Console'>)</font> <b>{</b>
                <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> old <font color='#5555FF'>=</font> src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[idx];
                src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[idx] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> eps;
                <font color='#BB00BB'>sigmoid</font><font face='Lucida Console'>(</font>dest, src<font face='Lucida Console'>)</font>;
                <font color='#0000FF'><u>float</u></font> result <font color='#5555FF'>=</font> <font color='#BB00BB'>dot</font><font face='Lucida Console'>(</font>gradient_input, dest<font face='Lucida Console'>)</font>;
                src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[idx] <font color='#5555FF'>=</font> old;
                <font color='#0000FF'>return</font> result;
            <b>}</b>;
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> eps <font color='#5555FF'>=</font> <font color='#979000'>0.01</font>;
            <font color='#0000FF'>return</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>f</font><font face='Lucida Console'>(</font><font color='#5555FF'>+</font>eps<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>f</font><font face='Lucida Console'>(</font><font color='#5555FF'>-</font>eps<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font face='Lucida Console'>(</font><font color='#979000'>2</font><font color='#5555FF'>*</font>eps<font face='Lucida Console'>)</font>;
        <b>}</b>;

        resizable_tensor src_grad;
        src_grad.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;
        src_grad <font color='#5555FF'>=</font> <font color='#979000'>0</font>;

        <font color='#BB00BB'>sigmoid</font><font face='Lucida Console'>(</font>dest, src<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>sigmoid_gradient</font><font face='Lucida Console'>(</font>src_grad, dest, gradient_input<font face='Lucida Console'>)</font>;

        <font color='#0000FF'>auto</font> grad_error <font color='#5555FF'>=</font> <font color='#BB00BB'>compare_gradients</font><font face='Lucida Console'>(</font>src_grad, grad_src<font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>src error: </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> grad_error;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>grad_error <font color='#5555FF'>&lt;</font> <font color='#979000'>0.001</font><font face='Lucida Console'>)</font>;
    <b>}</b>

    <font color='#0000FF'><u>void</u></font> <b><a name='test_softmax'></a>test_softmax</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#0000FF'>using</font> <font color='#0000FF'>namespace</font> dlib::tt;
        <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> nr <font color='#5555FF'>=</font> <font color='#979000'>3</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> nc <font color='#5555FF'>=</font> <font color='#979000'>3</font>;
        resizable_tensor <font color='#BB00BB'>src</font><font face='Lucida Console'>(</font><font color='#979000'>5</font>,<font color='#979000'>5</font>,nr,nr<font face='Lucida Console'>)</font>, <font color='#BB00BB'>dest</font><font face='Lucida Console'>(</font><font color='#979000'>5</font>,<font color='#979000'>5</font>,nr,nc<font face='Lucida Console'>)</font>, <font color='#BB00BB'>gradient_input</font><font face='Lucida Console'>(</font><font color='#979000'>5</font>,<font color='#979000'>5</font>,nr,nc<font face='Lucida Console'>)</font>;
        tt::tensor_rand rnd;
        rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;
        rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font>;
        <font color='#009900'>// fill like this as a test of the assignment operator.
</font>        gradient_input <font color='#5555FF'>=</font> matrix_cast<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font><font color='#BB00BB'>gaussian_randm</font><font face='Lucida Console'>(</font><font color='#979000'>5</font>,<font color='#979000'>5</font><font color='#5555FF'>*</font>nr<font color='#5555FF'>*</font>nc, <font color='#979000'>2</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;



        <font color='#0000FF'>auto</font> grad_src <font color='#5555FF'>=</font> [<font color='#5555FF'>&amp;</font>]<font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> idx<font face='Lucida Console'>)</font> <b>{</b>
            <font color='#0000FF'>auto</font> f <font color='#5555FF'>=</font> [<font color='#5555FF'>&amp;</font>]<font face='Lucida Console'>(</font><font color='#0000FF'><u>float</u></font> eps<font face='Lucida Console'>)</font> <b>{</b>
                <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> old <font color='#5555FF'>=</font> src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[idx];
                src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[idx] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> eps;
                tt::<font color='#BB00BB'>softmax</font><font face='Lucida Console'>(</font>dest, src<font face='Lucida Console'>)</font>;
                <font color='#0000FF'><u>float</u></font> result <font color='#5555FF'>=</font> <font color='#BB00BB'>dot</font><font face='Lucida Console'>(</font>gradient_input, dest<font face='Lucida Console'>)</font>;
                src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[idx] <font color='#5555FF'>=</font> old;
                <font color='#0000FF'>return</font> result;
            <b>}</b>;
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> eps <font color='#5555FF'>=</font> <font color='#979000'>0.01</font>;
            <font color='#0000FF'>return</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>f</font><font face='Lucida Console'>(</font><font color='#5555FF'>+</font>eps<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>f</font><font face='Lucida Console'>(</font><font color='#5555FF'>-</font>eps<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font face='Lucida Console'>(</font><font color='#979000'>2</font><font color='#5555FF'>*</font>eps<font face='Lucida Console'>)</font>;
        <b>}</b>;

        resizable_tensor src_grad;
        src_grad.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;
        src_grad <font color='#5555FF'>=</font> <font color='#979000'>0</font>;

        tt::<font color='#BB00BB'>softmax</font><font face='Lucida Console'>(</font>dest, src<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>softmax_gradient</font><font face='Lucida Console'>(</font>src_grad, dest, gradient_input<font face='Lucida Console'>)</font>;

        <font color='#0000FF'>auto</font> grad_error <font color='#5555FF'>=</font> <font color='#BB00BB'>compare_gradients</font><font face='Lucida Console'>(</font>src_grad, grad_src<font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>src error: </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> grad_error;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>grad_error <font color='#5555FF'>&lt;</font> <font color='#979000'>0.001</font><font face='Lucida Console'>)</font>;

<font color='#0000FF'>#ifdef</font> DLIB_USE_CUDA
        resizable_tensor src1 <font color='#5555FF'>=</font> src;
        resizable_tensor src2 <font color='#5555FF'>=</font> src;
        resizable_tensor dest1, dest2;
        dest1.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;
        dest2.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;
        cuda::<font color='#BB00BB'>softmax_all</font><font face='Lucida Console'>(</font>dest1, src1<font face='Lucida Console'>)</font>;
        cpu::<font color='#BB00BB'>softmax_all</font><font face='Lucida Console'>(</font>dest2, src2<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font>, <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>#endif</font>
    <b>}</b>

    <font color='#0000FF'><u>void</u></font> <b><a name='test_softmax_all'></a>test_softmax_all</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#0000FF'>using</font> <font color='#0000FF'>namespace</font> dlib::tt;
        <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> nr <font color='#5555FF'>=</font> <font color='#979000'>3</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> nc <font color='#5555FF'>=</font> <font color='#979000'>3</font>;
        resizable_tensor <font color='#BB00BB'>src</font><font face='Lucida Console'>(</font><font color='#979000'>5</font>,<font color='#979000'>5</font>,nr,nc<font face='Lucida Console'>)</font>, <font color='#BB00BB'>dest</font><font face='Lucida Console'>(</font><font color='#979000'>5</font>,<font color='#979000'>5</font>,nr,nc<font face='Lucida Console'>)</font>, <font color='#BB00BB'>gradient_input</font><font face='Lucida Console'>(</font><font color='#979000'>5</font>,<font color='#979000'>5</font>,nr,nc<font face='Lucida Console'>)</font>;
        tt::tensor_rand rnd;
        rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;
        rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font>;
        <font color='#009900'>// fill like this as a test of the assignment operator.
</font>        gradient_input <font color='#5555FF'>=</font> matrix_cast<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font><font color='#BB00BB'>gaussian_randm</font><font face='Lucida Console'>(</font><font color='#979000'>5</font>,<font color='#979000'>5</font><font color='#5555FF'>*</font>nr<font color='#5555FF'>*</font>nc, <font color='#979000'>2</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;



        <font color='#0000FF'>auto</font> grad_src <font color='#5555FF'>=</font> [<font color='#5555FF'>&amp;</font>]<font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> idx<font face='Lucida Console'>)</font> <b>{</b>
            <font color='#0000FF'>auto</font> f <font color='#5555FF'>=</font> [<font color='#5555FF'>&amp;</font>]<font face='Lucida Console'>(</font><font color='#0000FF'><u>float</u></font> eps<font face='Lucida Console'>)</font> <b>{</b>
                <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> old <font color='#5555FF'>=</font> src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[idx];
                src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[idx] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> eps;
                tt::<font color='#BB00BB'>softmax_all</font><font face='Lucida Console'>(</font>dest, src<font face='Lucida Console'>)</font>;
                <font color='#0000FF'><u>float</u></font> result <font color='#5555FF'>=</font> <font color='#BB00BB'>dot</font><font face='Lucida Console'>(</font>gradient_input, dest<font face='Lucida Console'>)</font>;
                src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[idx] <font color='#5555FF'>=</font> old;
                <font color='#0000FF'>return</font> result;
            <b>}</b>;
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> eps <font color='#5555FF'>=</font> <font color='#979000'>0.01</font>;
            <font color='#0000FF'>return</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>f</font><font face='Lucida Console'>(</font><font color='#5555FF'>+</font>eps<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>f</font><font face='Lucida Console'>(</font><font color='#5555FF'>-</font>eps<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font face='Lucida Console'>(</font><font color='#979000'>2</font><font color='#5555FF'>*</font>eps<font face='Lucida Console'>)</font>;
        <b>}</b>;

        resizable_tensor src_grad;
        src_grad.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;
        src_grad <font color='#5555FF'>=</font> <font color='#979000'>0</font>;

        tt::<font color='#BB00BB'>softmax_all</font><font face='Lucida Console'>(</font>dest, src<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>softmax_all_gradient</font><font face='Lucida Console'>(</font>src_grad, dest, gradient_input<font face='Lucida Console'>)</font>;

        <font color='#0000FF'>auto</font> grad_error <font color='#5555FF'>=</font> <font color='#BB00BB'>compare_gradients</font><font face='Lucida Console'>(</font>src_grad, grad_src<font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>src error: </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> grad_error;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>grad_error <font color='#5555FF'>&lt;</font> <font color='#979000'>0.001</font><font face='Lucida Console'>)</font>;

<font color='#0000FF'>#ifdef</font> DLIB_USE_CUDA
        resizable_tensor src1 <font color='#5555FF'>=</font> src;
        resizable_tensor src2 <font color='#5555FF'>=</font> src;
        resizable_tensor dest1, dest2;
        dest1.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;
        dest2.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;
        cuda::<font color='#BB00BB'>softmax_all</font><font face='Lucida Console'>(</font>dest1, src1<font face='Lucida Console'>)</font>;
        cpu::<font color='#BB00BB'>softmax_all</font><font face='Lucida Console'>(</font>dest2, src2<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font>, <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>#endif</font>
    <b>}</b>

    <font color='#0000FF'><u>void</u></font> <b><a name='test_mish'></a>test_mish</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
    <b>{</b>
<font color='#0000FF'>#ifdef</font> DLIB_USE_CUDA
        <font color='#009900'>// make sure that cuda::mish and cpu::mish return the same results
</font>        <font color='#0000FF'>using</font> <font color='#0000FF'>namespace</font> dlib::tt;
        <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> n <font color='#5555FF'>=</font> <font color='#979000'>4</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>5</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> nr <font color='#5555FF'>=</font> <font color='#979000'>3</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> nc <font color='#5555FF'>=</font> <font color='#979000'>3</font>;
        resizable_tensor <font color='#BB00BB'>src</font><font face='Lucida Console'>(</font>n,k,nr,nc<font face='Lucida Console'>)</font>;
        tt::tensor_rand rnd;
        rnd.<font color='#BB00BB'>fill_gaussian</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;

        resizable_tensor dest1, dest2;
        dest1.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;
        dest2.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;
        <font color='#009900'>// initialize to different values in order to make sure the output is actually changed
</font>        dest1 <font color='#5555FF'>=</font> <font color='#979000'>1</font>;
        dest2 <font color='#5555FF'>=</font> <font color='#979000'>2</font>;
        cuda::<font color='#BB00BB'>mish</font><font face='Lucida Console'>(</font>dest1, src<font face='Lucida Console'>)</font>;
        cpu::<font color='#BB00BB'>mish</font><font face='Lucida Console'>(</font>dest2, src<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font>, <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>#endif</font> <font color='#009900'>// DLIB_USE_CUDA
</font>    <b>}</b>

    <font color='#0000FF'><u>void</u></font> <b><a name='test_leaky_relu'></a>test_leaky_relu</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
    <b>{</b>
<font color='#0000FF'>#ifdef</font> DLIB_USE_CUDA
        <font color='#0000FF'>using</font> <font color='#0000FF'>namespace</font> dlib::tt;
        <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> n <font color='#5555FF'>=</font> <font color='#979000'>4</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>5</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> nr <font color='#5555FF'>=</font> <font color='#979000'>3</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> nc <font color='#5555FF'>=</font> <font color='#979000'>3</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> alpha <font color='#5555FF'>=</font> <font color='#979000'>0.01</font>;
        resizable_tensor <font color='#BB00BB'>src</font><font face='Lucida Console'>(</font>n, k, nr, nc<font face='Lucida Console'>)</font>;
        tt::tensor_rand rnd;
        rnd.<font color='#BB00BB'>fill_gaussian</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;
        resizable_tensor dest_cuda, dest_cpu;
        dest_cuda.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;
        dest_cpu.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;
        <font color='#009900'>// initialize to different values in order to make sure the output is actually changed
</font>        dest_cuda <font color='#5555FF'>=</font> <font color='#979000'>1</font>;
        dest_cpu <font color='#5555FF'>=</font> <font color='#979000'>2</font>;
        cuda::<font color='#BB00BB'>leaky_relu</font><font face='Lucida Console'>(</font>dest_cuda, src, alpha<font face='Lucida Console'>)</font>;
        cpu::<font color='#BB00BB'>leaky_relu</font><font face='Lucida Console'>(</font>dest_cpu, src, alpha<font face='Lucida Console'>)</font>;

        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest_cuda<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest_cpu<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>7</font>, <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest_cuda<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest_cpu<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>#endif</font> <font color='#009900'>// DLIB_USE_CUDA
</font>    <b>}</b>

    <font color='#0000FF'><u>void</u></font> <b><a name='test_clipped_relu'></a>test_clipped_relu</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
    <b>{</b>
<font color='#0000FF'>#ifdef</font> DLIB_USE_CUDA
        <font color='#0000FF'>using</font> <font color='#0000FF'>namespace</font> dlib::tt;
        <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> n <font color='#5555FF'>=</font> <font color='#979000'>4</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>5</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> nr <font color='#5555FF'>=</font> <font color='#979000'>3</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> nc <font color='#5555FF'>=</font> <font color='#979000'>3</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> ceiling <font color='#5555FF'>=</font> <font color='#979000'>6.0f</font>;
        resizable_tensor <font color='#BB00BB'>src</font><font face='Lucida Console'>(</font>n, k, nr, nc<font face='Lucida Console'>)</font>;
        tt::tensor_rand rnd;
        rnd.<font color='#BB00BB'>fill_gaussian</font><font face='Lucida Console'>(</font>src, <font color='#979000'>0</font>, <font color='#979000'>3</font><font face='Lucida Console'>)</font>;
        resizable_tensor dest_cuda, dest_cpu;
        dest_cuda.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;
        dest_cpu.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;
        <font color='#009900'>// initialize to different values in order to make sure the output is actually changed
</font>        dest_cuda <font color='#5555FF'>=</font> <font color='#979000'>1</font>;
        dest_cpu <font color='#5555FF'>=</font> <font color='#979000'>2</font>;
        cuda::<font color='#BB00BB'>clipped_relu</font><font face='Lucida Console'>(</font>dest_cuda, src, ceiling<font face='Lucida Console'>)</font>;
        cpu::<font color='#BB00BB'>clipped_relu</font><font face='Lucida Console'>(</font>dest_cpu, src, ceiling<font face='Lucida Console'>)</font>;
        <font color='#0000FF'>auto</font> error <font color='#5555FF'>=</font> <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest_cuda<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest_cpu<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>error <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>7</font>, "<font color='#CC0000'>error: </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> error<font face='Lucida Console'>)</font>;

        <font color='#009900'>// test gradients
</font>        resizable_tensor grad_cuda, grad_cpu, grad_input;
        grad_cuda.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;
        grad_cpu.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;
        grad_input.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;
        rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>grad_input<font face='Lucida Console'>)</font>;
        grad_cuda <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        grad_cpu <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        cuda::<font color='#BB00BB'>clipped_relu_gradient</font><font face='Lucida Console'>(</font>grad_cuda, dest_cuda, grad_input, ceiling<font face='Lucida Console'>)</font>;
        cpu::<font color='#BB00BB'>clipped_relu_gradient</font><font face='Lucida Console'>(</font>grad_cpu, dest_cpu, grad_input, ceiling<font face='Lucida Console'>)</font>;
        error <font color='#5555FF'>=</font> <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>grad_cuda<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>grad_cpu<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>error <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>7</font>, "<font color='#CC0000'>error: </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> error<font face='Lucida Console'>)</font>;
<font color='#0000FF'>#endif</font> <font color='#009900'>// DLIB_USE_CUDA
</font>    <b>}</b>

    <font color='#0000FF'><u>void</u></font> <b><a name='test_elu'></a>test_elu</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
    <b>{</b>
<font color='#0000FF'>#ifdef</font> DLIB_USE_CUDA
        <font color='#0000FF'>using</font> <font color='#0000FF'>namespace</font> dlib::tt;
        <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> n <font color='#5555FF'>=</font> <font color='#979000'>4</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>5</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> nr <font color='#5555FF'>=</font> <font color='#979000'>3</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> nc <font color='#5555FF'>=</font> <font color='#979000'>3</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> alpha <font color='#5555FF'>=</font> <font color='#979000'>1.0f</font>;
        resizable_tensor <font color='#BB00BB'>src</font><font face='Lucida Console'>(</font>n, k, nr, nc<font face='Lucida Console'>)</font>;
        tt::tensor_rand rnd;
        rnd.<font color='#BB00BB'>fill_gaussian</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;
        resizable_tensor dest_cuda, dest_cpu;
        dest_cuda.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;
        dest_cpu.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;
        <font color='#009900'>// initialize to different values in order to make sure the output is actually changed
</font>        dest_cuda <font color='#5555FF'>=</font> <font color='#979000'>1</font>;
        dest_cpu <font color='#5555FF'>=</font> <font color='#979000'>2</font>;
        cuda::<font color='#BB00BB'>elu</font><font face='Lucida Console'>(</font>dest_cuda, src, alpha<font face='Lucida Console'>)</font>;
        cpu::<font color='#BB00BB'>elu</font><font face='Lucida Console'>(</font>dest_cpu, src, alpha<font face='Lucida Console'>)</font>;
        <font color='#0000FF'>auto</font> error <font color='#5555FF'>=</font> <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest_cuda<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest_cpu<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>error <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>7</font>, "<font color='#CC0000'>error: </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> error<font face='Lucida Console'>)</font>;
        <font color='#009900'>// test gradients
</font>        resizable_tensor grad_cuda, grad_cpu, grad_input;
        grad_cuda.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;
        grad_cpu.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;
        grad_input.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;
        rnd.<font color='#BB00BB'>fill_gaussian</font><font face='Lucida Console'>(</font>grad_input<font face='Lucida Console'>)</font>;
        grad_cuda <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        grad_cpu <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        cuda::<font color='#BB00BB'>elu_gradient</font><font face='Lucida Console'>(</font>grad_cuda, dest_cuda, grad_input, alpha<font face='Lucida Console'>)</font>;
        cpu::<font color='#BB00BB'>elu_gradient</font><font face='Lucida Console'>(</font>grad_cpu, dest_cpu, grad_input, alpha<font face='Lucida Console'>)</font>;
        error <font color='#5555FF'>=</font> <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>grad_cuda<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>grad_cpu<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>error <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font>, "<font color='#CC0000'>error: </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> error<font face='Lucida Console'>)</font>;
<font color='#0000FF'>#endif</font> <font color='#009900'>// DLIB_USE_CUDA
</font>    <b>}</b>

    <font color='#0000FF'><u>void</u></font> <b><a name='test_gelu'></a>test_gelu</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
    <b>{</b>
<font color='#0000FF'>#ifdef</font> DLIB_USE_CUDA
        <font color='#009900'>// make sure that cuda::gelu and cpu::gelu return the same results
</font>        <font color='#0000FF'>using</font> <font color='#0000FF'>namespace</font> dlib::tt;
        <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> n <font color='#5555FF'>=</font> <font color='#979000'>4</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>5</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> nr <font color='#5555FF'>=</font> <font color='#979000'>3</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> nc <font color='#5555FF'>=</font> <font color='#979000'>3</font>;
        resizable_tensor <font color='#BB00BB'>src</font><font face='Lucida Console'>(</font>n,k,nr,nc<font face='Lucida Console'>)</font>;
        tt::tensor_rand rnd;
        rnd.<font color='#BB00BB'>fill_gaussian</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;

        resizable_tensor dest1, dest2;
        dest1.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;
        dest2.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;
        <font color='#009900'>// initialize to different values in order to make sure the output is actually changed
</font>        dest1 <font color='#5555FF'>=</font> <font color='#979000'>1</font>;
        dest2 <font color='#5555FF'>=</font> <font color='#979000'>2</font>;
        cuda::<font color='#BB00BB'>gelu</font><font face='Lucida Console'>(</font>dest1, src<font face='Lucida Console'>)</font>;
        cpu::<font color='#BB00BB'>gelu</font><font face='Lucida Console'>(</font>dest2, src<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font>, <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>#endif</font> <font color='#009900'>// DLIB_USE_CUDA
</font>    <b>}</b>

    <font color='#0000FF'><u>void</u></font> <b><a name='test_smelu'></a>test_smelu</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
    <b>{</b>
<font color='#0000FF'>#ifdef</font> DLIB_USE_CUDA
        <font color='#0000FF'>using</font> <font color='#0000FF'>namespace</font> dlib::tt;
        <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> n <font color='#5555FF'>=</font> <font color='#979000'>4</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>5</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> nr <font color='#5555FF'>=</font> <font color='#979000'>3</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> nc <font color='#5555FF'>=</font> <font color='#979000'>3</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> beta <font color='#5555FF'>=</font> <font color='#979000'>1</font>;
        resizable_tensor <font color='#BB00BB'>src</font><font face='Lucida Console'>(</font>n, k, nr, nc<font face='Lucida Console'>)</font>;
        tt::tensor_rand rnd;
        rnd.<font color='#BB00BB'>fill_gaussian</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;
        resizable_tensor dest_cuda, dest_cpu;
        dest_cuda.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;
        dest_cpu.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;
        <font color='#009900'>// initialize to different values in order to make sure the output is actually changed
</font>        dest_cuda <font color='#5555FF'>=</font> <font color='#979000'>1</font>;
        dest_cpu <font color='#5555FF'>=</font> <font color='#979000'>2</font>;
        cuda::<font color='#BB00BB'>smelu</font><font face='Lucida Console'>(</font>dest_cuda, src, beta<font face='Lucida Console'>)</font>;
        cpu::<font color='#BB00BB'>smelu</font><font face='Lucida Console'>(</font>dest_cpu, src, beta<font face='Lucida Console'>)</font>;

        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest_cuda<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest_cpu<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>7</font>, <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest_cuda<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest_cpu<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>#endif</font> <font color='#009900'>// DLIB_USE_CUDA
</font>    <b>}</b>

    <font color='#0000FF'><u>void</u></font> <b><a name='test_silu'></a>test_silu</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
    <b>{</b>
<font color='#0000FF'>#ifdef</font> DLIB_USE_CUDA
        <font color='#0000FF'>using</font> <font color='#0000FF'>namespace</font> dlib::tt;
        <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> n <font color='#5555FF'>=</font> <font color='#979000'>4</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>5</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> nr <font color='#5555FF'>=</font> <font color='#979000'>3</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> nc <font color='#5555FF'>=</font> <font color='#979000'>3</font>;
        resizable_tensor <font color='#BB00BB'>src</font><font face='Lucida Console'>(</font>n, k, nr, nc<font face='Lucida Console'>)</font>;
        tt::tensor_rand rnd;
        rnd.<font color='#BB00BB'>fill_gaussian</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;
        resizable_tensor dest_cuda, dest_cpu;
        dest_cuda.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;
        dest_cpu.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;
        <font color='#009900'>// initialize to different values in order to make sure the output is actually changed
</font>        dest_cuda <font color='#5555FF'>=</font> <font color='#979000'>1</font>;
        dest_cpu <font color='#5555FF'>=</font> <font color='#979000'>2</font>;
        cuda::<font color='#BB00BB'>silu</font><font face='Lucida Console'>(</font>dest_cuda, src<font face='Lucida Console'>)</font>;
        cpu::<font color='#BB00BB'>silu</font><font face='Lucida Console'>(</font>dest_cpu, src<font face='Lucida Console'>)</font>;

        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest_cuda<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest_cpu<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font>, <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest_cuda<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest_cpu<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>#endif</font> <font color='#009900'>// DLIB_USE_CUDA
</font>    <b>}</b>

    <font color='#0000FF'><u>void</u></font> <b><a name='test_batch_normalize'></a>test_batch_normalize</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#0000FF'>using</font> <font color='#0000FF'>namespace</font> dlib::tt;
        <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        resizable_tensor src, gamma, beta, dest, dest2, dest3, means, vars, gradient_input;
        src <font color='#5555FF'>=</font> matrix_cast<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font><font color='#BB00BB'>gaussian_randm</font><font face='Lucida Console'>(</font><font color='#979000'>5</font>,<font color='#979000'>5</font>, <font color='#979000'>0</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        gamma <font color='#5555FF'>=</font> matrix_cast<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font><font color='#BB00BB'>gaussian_randm</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,<font color='#979000'>5</font>, <font color='#979000'>1</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        beta <font color='#5555FF'>=</font> matrix_cast<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font><font color='#BB00BB'>gaussian_randm</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,<font color='#979000'>5</font>, <font color='#979000'>2</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        gradient_input <font color='#5555FF'>=</font> matrix_cast<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font><font color='#BB00BB'>gaussian_randm</font><font face='Lucida Console'>(</font><font color='#979000'>5</font>,<font color='#979000'>5</font>, <font color='#979000'>3</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

        gamma <font color='#5555FF'>=</font> <font color='#979000'>1</font>;
        beta <font color='#5555FF'>=</font> <font color='#979000'>0</font>;

        resizable_tensor running_means;
        resizable_tensor running_variances;
        <font color='#BB00BB'>batch_normalize</font><font face='Lucida Console'>(</font>DEFAULT_BATCH_NORM_EPS,dest, means, vars, <font color='#979000'>1</font>, running_means, running_variances, src, gamma, beta<font face='Lucida Console'>)</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> scale <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font>src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font face='Lucida Console'>(</font>src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#979000'>1.0</font><font face='Lucida Console'>)</font>;
        <font color='#009900'>// Turn back into biased variance estimate because that's how batch_normalize() works, so if we want to match it this is necessary.
</font>        running_variances <font color='#5555FF'>=</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>running_variances<font face='Lucida Console'>)</font><font color='#5555FF'>/</font>scale; 
        <font color='#BB00BB'>batch_normalize_inference</font><font face='Lucida Console'>(</font>DEFAULT_BATCH_NORM_EPS,dest2, src, gamma, beta, running_means, running_variances<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font>, <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        cpu::<font color='#BB00BB'>batch_normalize_inference</font><font face='Lucida Console'>(</font>DEFAULT_BATCH_NORM_EPS,dest3, src, gamma, beta, running_means, running_variances<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest3<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font>, <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest3<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;


        <font color='#0000FF'>auto</font> grad_src <font color='#5555FF'>=</font> [<font color='#5555FF'>&amp;</font>]<font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> idx<font face='Lucida Console'>)</font> <b>{</b>
            <font color='#0000FF'>auto</font> f <font color='#5555FF'>=</font> [<font color='#5555FF'>&amp;</font>]<font face='Lucida Console'>(</font><font color='#0000FF'><u>float</u></font> eps<font face='Lucida Console'>)</font> <b>{</b>
                <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> old <font color='#5555FF'>=</font> src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[idx];
                src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[idx] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> eps;
                <font color='#BB00BB'>batch_normalize</font><font face='Lucida Console'>(</font>DEFAULT_BATCH_NORM_EPS,dest, means, vars, <font color='#979000'>1</font>, running_means, running_variances, src, gamma, beta<font face='Lucida Console'>)</font>;
                <font color='#0000FF'><u>float</u></font> result <font color='#5555FF'>=</font> <font color='#BB00BB'>dot</font><font face='Lucida Console'>(</font>gradient_input, dest<font face='Lucida Console'>)</font>;
                src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[idx] <font color='#5555FF'>=</font> old;
                <font color='#0000FF'>return</font> result;
            <b>}</b>;
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> eps <font color='#5555FF'>=</font> <font color='#979000'>0.01</font>;
            <font color='#0000FF'>return</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>f</font><font face='Lucida Console'>(</font><font color='#5555FF'>+</font>eps<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>f</font><font face='Lucida Console'>(</font><font color='#5555FF'>-</font>eps<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font face='Lucida Console'>(</font><font color='#979000'>2</font><font color='#5555FF'>*</font>eps<font face='Lucida Console'>)</font>;
        <b>}</b>;
        <font color='#0000FF'>auto</font> grad_gamma <font color='#5555FF'>=</font> [<font color='#5555FF'>&amp;</font>]<font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> idx<font face='Lucida Console'>)</font> <b>{</b>
            <font color='#0000FF'>auto</font> f <font color='#5555FF'>=</font> [<font color='#5555FF'>&amp;</font>]<font face='Lucida Console'>(</font><font color='#0000FF'><u>float</u></font> eps<font face='Lucida Console'>)</font> <b>{</b>
                <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> old <font color='#5555FF'>=</font> gamma.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[idx];
                gamma.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[idx] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> eps;
                <font color='#BB00BB'>batch_normalize</font><font face='Lucida Console'>(</font>DEFAULT_BATCH_NORM_EPS,dest, means, vars, <font color='#979000'>1</font>, running_means, running_variances, src, gamma, beta<font face='Lucida Console'>)</font>;
                <font color='#0000FF'><u>float</u></font> result <font color='#5555FF'>=</font> <font color='#BB00BB'>dot</font><font face='Lucida Console'>(</font>gradient_input, dest<font face='Lucida Console'>)</font>;
                gamma.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[idx] <font color='#5555FF'>=</font> old;
                <font color='#0000FF'>return</font> result;
            <b>}</b>;
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> eps <font color='#5555FF'>=</font> <font color='#979000'>0.01</font>;
            <font color='#0000FF'>return</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>f</font><font face='Lucida Console'>(</font><font color='#5555FF'>+</font>eps<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>f</font><font face='Lucida Console'>(</font><font color='#5555FF'>-</font>eps<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font face='Lucida Console'>(</font><font color='#979000'>2</font><font color='#5555FF'>*</font>eps<font face='Lucida Console'>)</font>;
        <b>}</b>;
        <font color='#0000FF'>auto</font> grad_beta <font color='#5555FF'>=</font> [<font color='#5555FF'>&amp;</font>]<font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> idx<font face='Lucida Console'>)</font> <b>{</b>
            <font color='#0000FF'>auto</font> f <font color='#5555FF'>=</font> [<font color='#5555FF'>&amp;</font>]<font face='Lucida Console'>(</font><font color='#0000FF'><u>float</u></font> eps<font face='Lucida Console'>)</font> <b>{</b>
                <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> old <font color='#5555FF'>=</font> beta.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[idx];
                beta.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[idx] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> eps;
                <font color='#BB00BB'>batch_normalize</font><font face='Lucida Console'>(</font>DEFAULT_BATCH_NORM_EPS,dest, means, vars, <font color='#979000'>1</font>, running_means, running_variances, src, gamma, beta<font face='Lucida Console'>)</font>;
                <font color='#0000FF'><u>float</u></font> result <font color='#5555FF'>=</font> <font color='#BB00BB'>dot</font><font face='Lucida Console'>(</font>gradient_input, dest<font face='Lucida Console'>)</font>;
                beta.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[idx] <font color='#5555FF'>=</font> old;
                <font color='#0000FF'>return</font> result;
            <b>}</b>;
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> eps <font color='#5555FF'>=</font> <font color='#979000'>0.01</font>;
            <font color='#0000FF'>return</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>f</font><font face='Lucida Console'>(</font><font color='#5555FF'>+</font>eps<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>f</font><font face='Lucida Console'>(</font><font color='#5555FF'>-</font>eps<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font face='Lucida Console'>(</font><font color='#979000'>2</font><font color='#5555FF'>*</font>eps<font face='Lucida Console'>)</font>;
        <b>}</b>;

        resizable_tensor src_grad, gamma_grad, beta_grad;
        src_grad.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;
        gamma_grad.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>gamma<font face='Lucida Console'>)</font>;
        beta_grad.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>beta<font face='Lucida Console'>)</font>;
        src_grad <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        gamma_grad <font color='#5555FF'>=</font> <font color='#979000'>8</font>;
        beta_grad <font color='#5555FF'>=</font> <font color='#979000'>8</font>;

        <font color='#BB00BB'>batch_normalize_gradient</font><font face='Lucida Console'>(</font>DEFAULT_BATCH_NORM_EPS,gradient_input, means, vars, src, gamma, src_grad, gamma_grad, beta_grad<font face='Lucida Console'>)</font>;

        <font color='#0000FF'>auto</font> grad_error <font color='#5555FF'>=</font> <font color='#BB00BB'>compare_gradients</font><font face='Lucida Console'>(</font>src_grad, grad_src<font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>src error: </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> grad_error;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>grad_error <font color='#5555FF'>&lt;</font> <font color='#979000'>0.001</font><font face='Lucida Console'>)</font>;

        grad_error <font color='#5555FF'>=</font> <font color='#BB00BB'>compare_gradients</font><font face='Lucida Console'>(</font>gamma_grad, grad_gamma<font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>gamma error: </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> grad_error;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>grad_error <font color='#5555FF'>&lt;</font> <font color='#979000'>0.001</font><font face='Lucida Console'>)</font>;

        grad_error <font color='#5555FF'>=</font> <font color='#BB00BB'>compare_gradients</font><font face='Lucida Console'>(</font>beta_grad, grad_beta<font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>beta error: </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> grad_error;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>grad_error <font color='#5555FF'>&lt;</font> <font color='#979000'>0.001</font><font face='Lucida Console'>)</font>;
    <b>}</b>

    <font color='#0000FF'><u>void</u></font> <b><a name='test_batch_normalize_conv'></a>test_batch_normalize_conv</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#0000FF'>using</font> <font color='#0000FF'>namespace</font> dlib::tt;
        <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        resizable_tensor <font color='#BB00BB'>src</font><font face='Lucida Console'>(</font><font color='#979000'>5</font>,<font color='#979000'>5</font>,<font color='#979000'>4</font>,<font color='#979000'>4</font><font face='Lucida Console'>)</font>, gamma, beta, dest, dest2, dest3, means, vars, <font color='#BB00BB'>gradient_input</font><font face='Lucida Console'>(</font><font color='#979000'>5</font>,<font color='#979000'>5</font>,<font color='#979000'>4</font>,<font color='#979000'>4</font><font face='Lucida Console'>)</font>;
        tt::tensor_rand rnd;
        rnd.<font color='#BB00BB'>fill_gaussian</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;
        rnd.<font color='#BB00BB'>fill_gaussian</font><font face='Lucida Console'>(</font>gradient_input<font face='Lucida Console'>)</font>;
        gamma <font color='#5555FF'>=</font> matrix_cast<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font><font color='#BB00BB'>gaussian_randm</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,<font color='#979000'>5</font>, <font color='#979000'>1</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        beta <font color='#5555FF'>=</font> matrix_cast<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font><font color='#BB00BB'>gaussian_randm</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,<font color='#979000'>5</font>, <font color='#979000'>2</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

        gamma <font color='#5555FF'>=</font> <font color='#979000'>1</font>;
        beta <font color='#5555FF'>=</font> <font color='#979000'>0</font>;

        resizable_tensor running_means;
        resizable_tensor running_variances;
        <font color='#BB00BB'>batch_normalize_conv</font><font face='Lucida Console'>(</font>DEFAULT_BATCH_NORM_EPS,dest, means, vars, <font color='#979000'>1</font>, running_means, running_variances, src, gamma, beta<font face='Lucida Console'>)</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> scale <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font>src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>src.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>src.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font face='Lucida Console'>(</font>src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>src.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>src.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#979000'>1.0</font><font face='Lucida Console'>)</font>;
        <font color='#009900'>// Turn back into biased variance estimate because that's how
</font>        <font color='#009900'>// batch_normalize_conv() works, so if we want to match it this is necessary.
</font>        running_variances <font color='#5555FF'>=</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>running_variances<font face='Lucida Console'>)</font><font color='#5555FF'>/</font>scale; 
        <font color='#BB00BB'>batch_normalize_conv_inference</font><font face='Lucida Console'>(</font>DEFAULT_BATCH_NORM_EPS,dest2, src, gamma, beta, running_means, running_variances<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;
        cpu::<font color='#BB00BB'>batch_normalize_conv_inference</font><font face='Lucida Console'>(</font>DEFAULT_BATCH_NORM_EPS,dest3, src, gamma, beta, running_means, running_variances<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest3<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;


        <font color='#0000FF'>auto</font> grad_src <font color='#5555FF'>=</font> [<font color='#5555FF'>&amp;</font>]<font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> idx<font face='Lucida Console'>)</font> <b>{</b>
            <font color='#0000FF'>auto</font> f <font color='#5555FF'>=</font> [<font color='#5555FF'>&amp;</font>]<font face='Lucida Console'>(</font><font color='#0000FF'><u>float</u></font> eps<font face='Lucida Console'>)</font> <b>{</b>
                <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> old <font color='#5555FF'>=</font> src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[idx];
                src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[idx] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> eps;
                <font color='#BB00BB'>batch_normalize_conv</font><font face='Lucida Console'>(</font>DEFAULT_BATCH_NORM_EPS,dest, means, vars, <font color='#979000'>1</font>, running_means, running_variances, src, gamma, beta<font face='Lucida Console'>)</font>;
                <font color='#0000FF'><u>float</u></font> result <font color='#5555FF'>=</font> <font color='#BB00BB'>dot</font><font face='Lucida Console'>(</font>gradient_input, dest<font face='Lucida Console'>)</font>;
                src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[idx] <font color='#5555FF'>=</font> old;
                <font color='#0000FF'>return</font> result;
            <b>}</b>;
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> eps <font color='#5555FF'>=</font> <font color='#979000'>0.01</font>;
            <font color='#0000FF'>return</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>f</font><font face='Lucida Console'>(</font><font color='#5555FF'>+</font>eps<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>f</font><font face='Lucida Console'>(</font><font color='#5555FF'>-</font>eps<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font face='Lucida Console'>(</font><font color='#979000'>2</font><font color='#5555FF'>*</font>eps<font face='Lucida Console'>)</font>;
        <b>}</b>;
        <font color='#0000FF'>auto</font> grad_gamma <font color='#5555FF'>=</font> [<font color='#5555FF'>&amp;</font>]<font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> idx<font face='Lucida Console'>)</font> <b>{</b>
            <font color='#0000FF'>auto</font> f <font color='#5555FF'>=</font> [<font color='#5555FF'>&amp;</font>]<font face='Lucida Console'>(</font><font color='#0000FF'><u>float</u></font> eps<font face='Lucida Console'>)</font> <b>{</b>
                <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> old <font color='#5555FF'>=</font> gamma.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[idx];
                gamma.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[idx] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> eps;
                <font color='#BB00BB'>batch_normalize_conv</font><font face='Lucida Console'>(</font>DEFAULT_BATCH_NORM_EPS,dest, means, vars, <font color='#979000'>1</font>, running_means, running_variances, src, gamma, beta<font face='Lucida Console'>)</font>;
                <font color='#0000FF'><u>float</u></font> result <font color='#5555FF'>=</font> <font color='#BB00BB'>dot</font><font face='Lucida Console'>(</font>gradient_input, dest<font face='Lucida Console'>)</font>;
                gamma.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[idx] <font color='#5555FF'>=</font> old;
                <font color='#0000FF'>return</font> result;
            <b>}</b>;
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> eps <font color='#5555FF'>=</font> <font color='#979000'>0.01</font>;
            <font color='#0000FF'>return</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>f</font><font face='Lucida Console'>(</font><font color='#5555FF'>+</font>eps<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>f</font><font face='Lucida Console'>(</font><font color='#5555FF'>-</font>eps<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font face='Lucida Console'>(</font><font color='#979000'>2</font><font color='#5555FF'>*</font>eps<font face='Lucida Console'>)</font>;
        <b>}</b>;
        <font color='#0000FF'>auto</font> grad_beta <font color='#5555FF'>=</font> [<font color='#5555FF'>&amp;</font>]<font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> idx<font face='Lucida Console'>)</font> <b>{</b>
            <font color='#0000FF'>auto</font> f <font color='#5555FF'>=</font> [<font color='#5555FF'>&amp;</font>]<font face='Lucida Console'>(</font><font color='#0000FF'><u>float</u></font> eps<font face='Lucida Console'>)</font> <b>{</b>
                <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> old <font color='#5555FF'>=</font> beta.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[idx];
                beta.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[idx] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> eps;
                <font color='#BB00BB'>batch_normalize_conv</font><font face='Lucida Console'>(</font>DEFAULT_BATCH_NORM_EPS,dest, means, vars, <font color='#979000'>1</font>, running_means, running_variances, src, gamma, beta<font face='Lucida Console'>)</font>;
                <font color='#0000FF'><u>float</u></font> result <font color='#5555FF'>=</font> <font color='#BB00BB'>dot</font><font face='Lucida Console'>(</font>gradient_input, dest<font face='Lucida Console'>)</font>;
                beta.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[idx] <font color='#5555FF'>=</font> old;
                <font color='#0000FF'>return</font> result;
            <b>}</b>;
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> eps <font color='#5555FF'>=</font> <font color='#979000'>0.01</font>;
            <font color='#0000FF'>return</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>f</font><font face='Lucida Console'>(</font><font color='#5555FF'>+</font>eps<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>f</font><font face='Lucida Console'>(</font><font color='#5555FF'>-</font>eps<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font face='Lucida Console'>(</font><font color='#979000'>2</font><font color='#5555FF'>*</font>eps<font face='Lucida Console'>)</font>;
        <b>}</b>;


        resizable_tensor src_grad, gamma_grad, beta_grad;
        src_grad.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;
        gamma_grad.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>gamma<font face='Lucida Console'>)</font>;
        beta_grad.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>beta<font face='Lucida Console'>)</font>;
        src_grad <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        gamma_grad <font color='#5555FF'>=</font> <font color='#979000'>9</font>;
        beta_grad <font color='#5555FF'>=</font> <font color='#979000'>9</font>;

        <font color='#BB00BB'>batch_normalize_conv_gradient</font><font face='Lucida Console'>(</font>DEFAULT_BATCH_NORM_EPS,gradient_input, means, vars, src, gamma, src_grad, gamma_grad, beta_grad<font face='Lucida Console'>)</font>;


        <font color='#0000FF'>auto</font> grad_error <font color='#5555FF'>=</font> <font color='#BB00BB'>compare_gradients</font><font face='Lucida Console'>(</font>src_grad, grad_src<font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>src error: </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> grad_error;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>grad_error <font color='#5555FF'>&lt;</font> <font color='#979000'>0.001</font><font face='Lucida Console'>)</font>;

        grad_error <font color='#5555FF'>=</font> <font color='#BB00BB'>compare_gradients</font><font face='Lucida Console'>(</font>gamma_grad, grad_gamma<font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>gamma error: </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> grad_error;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>grad_error <font color='#5555FF'>&lt;</font> <font color='#979000'>0.001</font><font face='Lucida Console'>)</font>;

        grad_error <font color='#5555FF'>=</font> <font color='#BB00BB'>compare_gradients</font><font face='Lucida Console'>(</font>beta_grad, grad_beta<font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>beta error: </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> grad_error;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>grad_error <font color='#5555FF'>&lt;</font> <font color='#979000'>0.001</font><font face='Lucida Console'>)</font>;

    <b>}</b>

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'><u>void</u></font> <b><a name='test_layer_normalize'></a>test_layer_normalize</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
    <b>{</b>
        resizable_tensor <font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font color='#979000'>2</font>, <font color='#979000'>3</font>, <font color='#979000'>4</font>, <font color='#979000'>5</font><font face='Lucida Console'>)</font>;
        resizable_tensor <font color='#BB00BB'>y_cpu</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font>;
        tt::tensor_rand <font color='#BB00BB'>rnd</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font>;
        rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font>;
        resizable_tensor <font color='#BB00BB'>means_cpu</font><font face='Lucida Console'>(</font>x.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>, <font color='#BB00BB'>invstds_cpu</font><font face='Lucida Console'>(</font>x.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        resizable_tensor <font color='#BB00BB'>gamma</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>, x.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, x.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, x.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>, <font color='#BB00BB'>beta</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>, x.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, x.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, x.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        gamma <font color='#5555FF'>=</font> <font color='#979000'>1</font>;
        beta <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> eps <font color='#5555FF'>=</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font>;
        cpu::<font color='#BB00BB'>layer_normalize</font><font face='Lucida Console'>(</font>eps, y_cpu, means_cpu, invstds_cpu, x, gamma, beta<font face='Lucida Console'>)</font>;
        <font color='#009900'>// check that the mean and var per sample are 0 and 1
</font>        <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> p <font color='#5555FF'>=</font> y_cpu.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> n <font color='#5555FF'>=</font> <font color='#979000'>0</font>; n <font color='#5555FF'>&lt;</font> y_cpu.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>n<font face='Lucida Console'>)</font>
        <b>{</b>
            running_stats<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font> rs;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'>&lt;</font> y_cpu.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> r <font color='#5555FF'>=</font> <font color='#979000'>0</font>; r <font color='#5555FF'>&lt;</font> y_cpu.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>r<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> c <font color='#5555FF'>=</font> <font color='#979000'>0</font>; c <font color='#5555FF'>&lt;</font> y_cpu.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>c<font face='Lucida Console'>)</font>
                    <b>{</b>
                        rs.<font color='#BB00BB'>add</font><font face='Lucida Console'>(</font>p[<font color='#BB00BB'>tensor_index</font><font face='Lucida Console'>(</font>y_cpu, n, k, r, c<font face='Lucida Console'>)</font>]<font face='Lucida Console'>)</font>;
                    <b>}</b>
                <b>}</b>
            <b>}</b>
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>::std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>rs.<font color='#BB00BB'>mean</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>::std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>rs.<font color='#BB00BB'>stddev</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> <font color='#979000'>1.0f</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>0.01</font><font face='Lucida Console'>)</font>;
        <b>}</b>
        <font color='#009900'>// check that the CPU and the CUDA implementation are equivalent
</font><font color='#0000FF'>#if</font> DLIB_USE_CUDA
        resizable_tensor <font color='#BB00BB'>y_cuda</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font>;
        resizable_tensor <font color='#BB00BB'>means_cuda</font><font face='Lucida Console'>(</font>x.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>, <font color='#BB00BB'>invstds_cuda</font><font face='Lucida Console'>(</font>x.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        cuda::<font color='#BB00BB'>layer_normalize</font><font face='Lucida Console'>(</font>eps, y_cuda, means_cuda, invstds_cuda, x, gamma, beta<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>y_cpu<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>y_cuda<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>means_cpu<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>means_cuda<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>invstds_cpu<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>invstds_cuda<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;
        resizable_tensor <font color='#BB00BB'>gradient_input</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font>;
        resizable_tensor <font color='#BB00BB'>src_grad_cpu</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font>, <font color='#BB00BB'>gamma_grad_cpu</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>, x.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, x.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, x.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>, <font color='#BB00BB'>beta_grad_cpu</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>, x.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, x.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, x.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        resizable_tensor <font color='#BB00BB'>src_grad_cuda</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font>, <font color='#BB00BB'>gamma_grad_cuda</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>, x.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, x.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, x.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>, <font color='#BB00BB'>beta_grad_cuda</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>, x.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, x.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, x.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        rnd.<font color='#BB00BB'>fill_gaussian</font><font face='Lucida Console'>(</font>gradient_input<font face='Lucida Console'>)</font>;
        src_grad_cpu <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        src_grad_cuda <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        cpu::<font color='#BB00BB'>layer_normalize_gradient</font><font face='Lucida Console'>(</font>eps, gradient_input, means_cpu, invstds_cpu, x, gamma, src_grad_cpu, gamma_grad_cpu, beta_grad_cpu<font face='Lucida Console'>)</font>;
        cuda::<font color='#BB00BB'>layer_normalize_gradient</font><font face='Lucida Console'>(</font>eps, gradient_input, means_cuda, invstds_cuda, x, gamma, src_grad_cuda, gamma_grad_cuda, beta_grad_cuda<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>src_grad_cpu<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>src_grad_cuda<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>gamma_grad_cpu<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>gamma_grad_cuda<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>beta_grad_cpu<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>beta_grad_cuda<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>#endif</font>
    <b>}</b>

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'><u>void</u></font> <b><a name='test_basic_tensor_ops'></a>test_basic_tensor_ops</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#0000FF'>using</font> <font color='#0000FF'>namespace</font> dlib::tt;
        <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        resizable_tensor dest, <font color='#BB00BB'>src</font><font face='Lucida Console'>(</font><font color='#979000'>3</font>,<font color='#979000'>4</font><font face='Lucida Console'>)</font>, <font color='#BB00BB'>A</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,<font color='#979000'>4</font><font face='Lucida Console'>)</font>, <font color='#BB00BB'>B</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,<font color='#979000'>4</font><font face='Lucida Console'>)</font>;
        src <font color='#5555FF'>=</font> <font color='#979000'>2</font>;
        dest.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>affine_transform</font><font face='Lucida Console'>(</font>dest, src, <font color='#979000'>2</font>, <font color='#979000'>3</font><font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font>;
        matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font> <font color='#BB00BB'>truth1</font><font face='Lucida Console'>(</font><font color='#979000'>3</font>,<font color='#979000'>4</font><font face='Lucida Console'>)</font>, <font color='#BB00BB'>truth2</font><font face='Lucida Console'>(</font><font color='#979000'>3</font>,<font color='#979000'>4</font><font face='Lucida Console'>)</font>;

        truth1 <font color='#5555FF'>=</font> <font color='#979000'>2</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>truth1<font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;
        src <font color='#5555FF'>*</font><font color='#5555FF'>=</font> <font color='#979000'>2</font>;
        truth1 <font color='#5555FF'>=</font> <font color='#979000'>4</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>truth1<font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;
        src <font color='#5555FF'>=</font> <font color='#979000'>2</font>;


        truth1 <font color='#5555FF'>=</font> <font color='#979000'>7</font>;
        truth2 <font color='#5555FF'>=</font> <font color='#979000'>7</font>, <font color='#979000'>10</font>,  <font color='#979000'>7</font>,  <font color='#979000'>7</font>,
        <font color='#979000'>7</font>, <font color='#979000'>10</font>,  <font color='#979000'>7</font>,  <font color='#979000'>7</font>,
        <font color='#979000'>7</font>, <font color='#979000'>10</font>,  <font color='#979000'>7</font>,  <font color='#979000'>7</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>truth1<font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;

        A <font color='#5555FF'>=</font> <font color='#979000'>2</font>;
        B <font color='#5555FF'>=</font> <font color='#979000'>3</font>;
        A.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[<font color='#979000'>1</font>] <font color='#5555FF'>=</font> <font color='#979000'>3</font>;
        B.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[<font color='#979000'>1</font>] <font color='#5555FF'>=</font> <font color='#979000'>4</font>;
        dest <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        <font color='#BB00BB'>affine_transform</font><font face='Lucida Console'>(</font>dest, src, A, B<font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>truth2<font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;

        A <font color='#5555FF'>=</font> matrix_cast<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font><font color='#BB00BB'>gaussian_randm</font><font face='Lucida Console'>(</font><font color='#979000'>3</font>,<font color='#979000'>4</font>, <font color='#979000'>1</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        B <font color='#5555FF'>=</font> matrix_cast<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font><font color='#BB00BB'>gaussian_randm</font><font face='Lucida Console'>(</font><font color='#979000'>3</font>,<font color='#979000'>4</font>, <font color='#979000'>2</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>affine_transform</font><font face='Lucida Console'>(</font>dest, src, A, B<font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font>;
        matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font> truth3 <font color='#5555FF'>=</font> <font color='#BB00BB'>pointwise_multiply</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>, <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>truth3<font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;

        matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font> truth4 <font color='#5555FF'>=</font> <font color='#BB00BB'>pointwise_multiply</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font>, <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        tt::<font color='#BB00BB'>multiply</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, A, A, B<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>truth4<font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;
        truth4 <font color='#5555FF'>=</font> <font color='#BB00BB'>pointwise_multiply</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font>, <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font>;
        tt::<font color='#BB00BB'>multiply</font><font face='Lucida Console'>(</font><font color='#979000'>true</font>, A, A, B<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>truth4<font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;

        matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font> truth5 <font color='#5555FF'>=</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font> <font color='#5555FF'>&gt;</font> <font color='#979000'>0.1</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> truth5;
        <font color='#BB00BB'>threshold</font><font face='Lucida Console'>(</font>B, <font color='#979000'>0.1</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>truth5<font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;

        <font color='#0000FF'><u>int</u></font> cnt <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        <font color='#0000FF'>for</font><font face='Lucida Console'>(</font><font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font> x : A<font face='Lucida Console'>)</font>
            x <font color='#5555FF'>=</font> cnt<font color='#5555FF'>+</font><font color='#5555FF'>+</font>;

        truth1.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font><font color='#979000'>2</font>,<font color='#979000'>2</font><font face='Lucida Console'>)</font>;
        truth2.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font><font color='#979000'>2</font>,<font color='#979000'>2</font><font face='Lucida Console'>)</font>;
        truth3.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font><font color='#979000'>2</font>,<font color='#979000'>2</font><font face='Lucida Console'>)</font>;
        truth1 <font color='#5555FF'>=</font> <font color='#979000'>0</font>,<font color='#979000'>1</font>,<font color='#979000'>2</font>,<font color='#979000'>3</font>;
        truth2 <font color='#5555FF'>=</font> <font color='#979000'>4</font>,<font color='#979000'>5</font>,<font color='#979000'>6</font>,<font color='#979000'>7</font>;
        truth3 <font color='#5555FF'>=</font> <font color='#979000'>8</font>,<font color='#979000'>9</font>,<font color='#979000'>10</font>,<font color='#979000'>11</font>;

        alias_tensor <font color='#BB00BB'>at</font><font face='Lucida Console'>(</font><font color='#979000'>2</font>,<font color='#979000'>2</font><font face='Lucida Console'>)</font>;
        <font color='#0000FF'>auto</font> A0 <font color='#5555FF'>=</font> <font color='#BB00BB'>at</font><font face='Lucida Console'>(</font>A,<font color='#979000'>0</font><font face='Lucida Console'>)</font>;
        <font color='#0000FF'>auto</font> A4 <font color='#5555FF'>=</font> <font color='#BB00BB'>at</font><font face='Lucida Console'>(</font>A,<font color='#979000'>4</font><font face='Lucida Console'>)</font>;
        <font color='#0000FF'>auto</font> A8 <font color='#5555FF'>=</font> <font color='#BB00BB'>at</font><font face='Lucida Console'>(</font><font color='#0000FF'>const_cast</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>const</font> resizable_tensor<font color='#5555FF'>&amp;</font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font>,<font color='#979000'>8</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A0<font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> truth1<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font><font color='#BB00BB'>at</font><font face='Lucida Console'>(</font>A,<font color='#979000'>4</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> truth2<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A8<font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> truth3<font face='Lucida Console'>)</font>;

        A4 <font color='#5555FF'>+</font><font color='#5555FF'>=</font> uniform_matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font><font color='#979000'>2</font>,<font color='#979000'>2</font>,<font color='#979000'>2</font><font face='Lucida Console'>)</font>;
        truth2 <font color='#5555FF'>+</font><font color='#5555FF'>=</font> <font color='#979000'>2</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A4<font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> truth2<font face='Lucida Console'>)</font>;
        truth1 <font color='#5555FF'>=</font> <font color='#BB00BB'>trans</font><font face='Lucida Console'>(</font><font color='#BB00BB'>reshape_to_column_vector</font><font face='Lucida Console'>(</font>truth1<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        truth2 <font color='#5555FF'>=</font> <font color='#BB00BB'>trans</font><font face='Lucida Console'>(</font><font color='#BB00BB'>reshape_to_column_vector</font><font face='Lucida Console'>(</font>truth2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        truth3 <font color='#5555FF'>=</font> <font color='#BB00BB'>trans</font><font face='Lucida Console'>(</font><font color='#BB00BB'>reshape_to_column_vector</font><font face='Lucida Console'>(</font>truth3<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#BB00BB'>join_cols</font><font face='Lucida Console'>(</font>truth1,<font color='#BB00BB'>join_cols</font><font face='Lucida Console'>(</font>truth2,truth3<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

        <font color='#BB00BB'>affine_transform</font><font face='Lucida Console'>(</font>A,A,<font color='#979000'>1</font>,<font color='#979000'>2</font><font face='Lucida Console'>)</font>;
        truth1 <font color='#5555FF'>+</font><font color='#5555FF'>=</font> <font color='#979000'>2</font>;
        truth2 <font color='#5555FF'>+</font><font color='#5555FF'>=</font> <font color='#979000'>2</font>;
        truth3 <font color='#5555FF'>+</font><font color='#5555FF'>=</font> <font color='#979000'>2</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font><font color='#BB00BB'>at</font><font face='Lucida Console'>(</font>A,<font color='#979000'>4</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#BB00BB'>reshape</font><font face='Lucida Console'>(</font>truth2,<font color='#979000'>2</font>,<font color='#979000'>2</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#BB00BB'>join_cols</font><font face='Lucida Console'>(</font>truth1,<font color='#BB00BB'>join_cols</font><font face='Lucida Console'>(</font>truth2,truth3<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

        <b>{</b>
            resizable_tensor <font color='#BB00BB'>dest</font><font face='Lucida Console'>(</font><font color='#979000'>3</font>,<font color='#979000'>4</font><font face='Lucida Console'>)</font>;
            resizable_tensor A, B;
            A <font color='#5555FF'>=</font> dest;
            B <font color='#5555FF'>=</font> dest;

            tensor_rand rnd;
            rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font>;
            rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font>;
            rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font>;

            dest.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,<font color='#979000'>4</font><font face='Lucida Console'>)</font>;

            tt::<font color='#BB00BB'>multiply</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, dest, A, B<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>sum_rows</font><font face='Lucida Console'>(</font><font color='#BB00BB'>pointwise_multiply</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font>,<font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font><font face='Lucida Console'>)</font>; 

            A.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,<font color='#979000'>4</font><font face='Lucida Console'>)</font>;
            rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font>;
            matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font> AA <font color='#5555FF'>=</font> <font color='#BB00BB'>join_cols</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font>,<font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>; AA <font color='#5555FF'>=</font> <font color='#BB00BB'>join_cols</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font>,AA<font face='Lucida Console'>)</font>;

            tt::<font color='#BB00BB'>multiply</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, dest, A, B<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>sum_rows</font><font face='Lucida Console'>(</font><font color='#BB00BB'>pointwise_multiply</font><font face='Lucida Console'>(</font>AA,<font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font><font face='Lucida Console'>)</font>; 

            tt::<font color='#BB00BB'>multiply</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, dest, B, A<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>sum_rows</font><font face='Lucida Console'>(</font><font color='#BB00BB'>pointwise_multiply</font><font face='Lucida Console'>(</font>AA,<font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font><font face='Lucida Console'>)</font>; 
            matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font> prevdest <font color='#5555FF'>=</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font>;
            tt::<font color='#BB00BB'>multiply</font><font face='Lucida Console'>(</font><font color='#979000'>true</font>, dest, B, A<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font><font color='#5555FF'>-</font>prevdest<font color='#5555FF'>-</font><font color='#BB00BB'>sum_rows</font><font face='Lucida Console'>(</font><font color='#BB00BB'>pointwise_multiply</font><font face='Lucida Console'>(</font>AA,<font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font><font face='Lucida Console'>)</font>; 

            dest.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font><font color='#979000'>3</font>,<font color='#979000'>4</font><font face='Lucida Console'>)</font>;
            tt::<font color='#BB00BB'>multiply</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, dest, B, A<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>pointwise_multiply</font><font face='Lucida Console'>(</font>AA,<font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font><font face='Lucida Console'>)</font>; 
            prevdest <font color='#5555FF'>=</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font>;
            tt::<font color='#BB00BB'>multiply</font><font face='Lucida Console'>(</font><font color='#979000'>true</font>, dest, B, A<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font><font color='#5555FF'>-</font>prevdest<font color='#5555FF'>-</font><font color='#BB00BB'>pointwise_multiply</font><font face='Lucida Console'>(</font>AA,<font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font><font face='Lucida Console'>)</font>; 

            tt::<font color='#BB00BB'>multiply</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, dest, A, B<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>pointwise_multiply</font><font face='Lucida Console'>(</font>AA,<font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font><font face='Lucida Console'>)</font>; 
            prevdest <font color='#5555FF'>=</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font>;
            tt::<font color='#BB00BB'>multiply</font><font face='Lucida Console'>(</font><font color='#979000'>true</font>, dest, B, A<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font><font color='#5555FF'>-</font>prevdest<font color='#5555FF'>-</font><font color='#BB00BB'>pointwise_multiply</font><font face='Lucida Console'>(</font>AA,<font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font><font face='Lucida Console'>)</font>; 
        <b>}</b>

        <b>{</b>
            resizable_tensor A, B, truth;
            A.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font><font color='#979000'>2</font>,<font color='#979000'>3</font>,<font color='#979000'>4</font>,<font color='#979000'>5</font><font face='Lucida Console'>)</font>;
            truth.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font>;
            B.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font>;

            A <font color='#5555FF'>=</font> <font color='#979000'>4</font>;
            B <font color='#5555FF'>=</font> <font color='#979000'>1</font>;
            truth <font color='#5555FF'>=</font> <font color='#979000'>1</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font><font color='#5555FF'>-</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>truth<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>memcpy</font><font face='Lucida Console'>(</font>A, truth<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font><font color='#5555FF'>-</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>truth<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;

            A <font color='#5555FF'>=</font> <font color='#979000'>4</font>;
            A.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            B.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>memcpy</font><font face='Lucida Console'>(</font>A, truth<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font><font color='#5555FF'>-</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>truth<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;

<font color='#0000FF'>#ifdef</font> DLIB_USE_CUDA
            A <font color='#5555FF'>=</font> <font color='#979000'>4</font>;
            A.<font color='#BB00BB'>device</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            B.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>memcpy</font><font face='Lucida Console'>(</font>A, truth<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font><font color='#5555FF'>-</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>truth<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;

            A <font color='#5555FF'>=</font> <font color='#979000'>4</font>;
            A.<font color='#BB00BB'>device</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            B.<font color='#BB00BB'>device</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>memcpy</font><font face='Lucida Console'>(</font>A, truth<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font><font color='#5555FF'>-</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>truth<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;

            A <font color='#5555FF'>=</font> <font color='#979000'>4</font>;
            A.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            B.<font color='#BB00BB'>device</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>memcpy</font><font face='Lucida Console'>(</font>A, truth<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font><font color='#5555FF'>-</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>truth<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;

            A <font color='#5555FF'>=</font> <font color='#979000'>4</font>;
            A.<font color='#BB00BB'>host_write_only</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            B.<font color='#BB00BB'>device</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>memcpy</font><font face='Lucida Console'>(</font>A, truth<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font><font color='#5555FF'>-</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>truth<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>#endif</font>
        <b>}</b>

        <b>{</b>
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>int</u></font> nr <font color='#5555FF'>=</font> <font color='#979000'>5</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>int</u></font> nc <font color='#5555FF'>=</font> <font color='#979000'>6</font>;
            tensor_rand rnd;
            resizable_tensor <font color='#BB00BB'>out1</font><font face='Lucida Console'>(</font>nr,nc<font face='Lucida Console'>)</font>, <font color='#BB00BB'>m</font><font face='Lucida Console'>(</font>nr,nc<font face='Lucida Console'>)</font>, <font color='#BB00BB'>v</font><font face='Lucida Console'>(</font>nc<font face='Lucida Console'>)</font>, out2;
            rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>out1<font face='Lucida Console'>)</font>;
            rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>m<font face='Lucida Console'>)</font>;
            rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>v<font face='Lucida Console'>)</font>;

            tt::<font color='#BB00BB'>scale_columns</font><font face='Lucida Console'>(</font>out1, m, v<font face='Lucida Console'>)</font>;
            out2 <font color='#5555FF'>=</font> <font color='#BB00BB'>scale_columns</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>m<font face='Lucida Console'>)</font>, <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>v<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>out1<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>out2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        <b>{</b>
            resizable_tensor A, B;
            A.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font><font color='#979000'>11</font><font face='Lucida Console'>)</font>;
            B.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font>;

            A <font color='#5555FF'>=</font> <font color='#979000'>4</font>;
            B <font color='#5555FF'>=</font> <font color='#979000'>1</font>;
            matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font> truth;


            alias_tensor <font color='#BB00BB'>at</font><font face='Lucida Console'>(</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;
            A <font color='#5555FF'>=</font> <font color='#979000'>4</font>;
            A.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            B.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <b>{</b>
                <font color='#009900'>// non-aliasing test
</font>                <font color='#0000FF'>auto</font> aA <font color='#5555FF'>=</font> <font color='#BB00BB'>at</font><font face='Lucida Console'>(</font>A,<font color='#979000'>5</font><font face='Lucida Console'>)</font>;
                <font color='#0000FF'>auto</font> aB <font color='#5555FF'>=</font> <font color='#BB00BB'>at</font><font face='Lucida Console'>(</font>B,<font color='#979000'>5</font><font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>memcpy</font><font face='Lucida Console'>(</font>aA, aB<font face='Lucida Console'>)</font>;
                truth <font color='#5555FF'>=</font> <b>{</b><font color='#979000'>4</font>,<font color='#979000'>4</font>,<font color='#979000'>4</font>,<font color='#979000'>4</font>,<font color='#979000'>4</font>,  <font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>, <font color='#979000'>4</font><b>}</b>;
                <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font><font color='#5555FF'>-</font> truth<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;
            <b>}</b>
            <b>{</b>
                <font color='#009900'>// aliasing test
</font>                <font color='#0000FF'>auto</font> aA <font color='#5555FF'>=</font> <font color='#BB00BB'>at</font><font face='Lucida Console'>(</font>A,<font color='#979000'>1</font><font face='Lucida Console'>)</font>;
                <font color='#0000FF'>auto</font> aB <font color='#5555FF'>=</font> <font color='#BB00BB'>at</font><font face='Lucida Console'>(</font>A,<font color='#979000'>6</font><font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>memcpy</font><font face='Lucida Console'>(</font>aA, aB<font face='Lucida Console'>)</font>;
                truth <font color='#5555FF'>=</font> <b>{</b><font color='#979000'>4</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,  <font color='#979000'>4</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>, <font color='#979000'>4</font><b>}</b>;
                <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font><font color='#5555FF'>-</font> truth<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;
            <b>}</b>


<font color='#0000FF'>#ifdef</font> DLIB_USE_CUDA
            A <font color='#5555FF'>=</font> <font color='#979000'>4</font>;
            A.<font color='#BB00BB'>device</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            B.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <b>{</b>
                <font color='#009900'>// non-aliasing test
</font>                <font color='#0000FF'>auto</font> aA <font color='#5555FF'>=</font> <font color='#BB00BB'>at</font><font face='Lucida Console'>(</font>A,<font color='#979000'>5</font><font face='Lucida Console'>)</font>;
                <font color='#0000FF'>auto</font> aB <font color='#5555FF'>=</font> <font color='#BB00BB'>at</font><font face='Lucida Console'>(</font>B,<font color='#979000'>5</font><font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>memcpy</font><font face='Lucida Console'>(</font>aA, aB<font face='Lucida Console'>)</font>;
                truth <font color='#5555FF'>=</font> <b>{</b><font color='#979000'>4</font>,<font color='#979000'>4</font>,<font color='#979000'>4</font>,<font color='#979000'>4</font>,<font color='#979000'>4</font>,  <font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>, <font color='#979000'>4</font><b>}</b>;
                <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font><font color='#5555FF'>-</font> truth<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;
            <b>}</b>
            <b>{</b>
                <font color='#009900'>// aliasing test
</font>                <font color='#0000FF'>auto</font> aA <font color='#5555FF'>=</font> <font color='#BB00BB'>at</font><font face='Lucida Console'>(</font>A,<font color='#979000'>1</font><font face='Lucida Console'>)</font>;
                <font color='#0000FF'>auto</font> aB <font color='#5555FF'>=</font> <font color='#BB00BB'>at</font><font face='Lucida Console'>(</font>A,<font color='#979000'>6</font><font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>memcpy</font><font face='Lucida Console'>(</font>aA, aB<font face='Lucida Console'>)</font>;
                truth <font color='#5555FF'>=</font> <b>{</b><font color='#979000'>4</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,  <font color='#979000'>4</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>, <font color='#979000'>4</font><b>}</b>;
                <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font><font color='#5555FF'>-</font> truth<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;
            <b>}</b>


            A <font color='#5555FF'>=</font> <font color='#979000'>4</font>;
            A.<font color='#BB00BB'>device</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            B.<font color='#BB00BB'>device</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <b>{</b>
                <font color='#009900'>// non-aliasing test
</font>                <font color='#0000FF'>auto</font> aA <font color='#5555FF'>=</font> <font color='#BB00BB'>at</font><font face='Lucida Console'>(</font>A,<font color='#979000'>5</font><font face='Lucida Console'>)</font>;
                <font color='#0000FF'>auto</font> aB <font color='#5555FF'>=</font> <font color='#BB00BB'>at</font><font face='Lucida Console'>(</font>B,<font color='#979000'>5</font><font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>memcpy</font><font face='Lucida Console'>(</font>aA, aB<font face='Lucida Console'>)</font>;
                truth <font color='#5555FF'>=</font> <b>{</b><font color='#979000'>4</font>,<font color='#979000'>4</font>,<font color='#979000'>4</font>,<font color='#979000'>4</font>,<font color='#979000'>4</font>,  <font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>, <font color='#979000'>4</font><b>}</b>;
                <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font><font color='#5555FF'>-</font> truth<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;
            <b>}</b>
            <b>{</b>
                <font color='#009900'>// aliasing test
</font>                <font color='#0000FF'>auto</font> aA <font color='#5555FF'>=</font> <font color='#BB00BB'>at</font><font face='Lucida Console'>(</font>A,<font color='#979000'>1</font><font face='Lucida Console'>)</font>;
                <font color='#0000FF'>auto</font> aB <font color='#5555FF'>=</font> <font color='#BB00BB'>at</font><font face='Lucida Console'>(</font>A,<font color='#979000'>6</font><font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>memcpy</font><font face='Lucida Console'>(</font>aA, aB<font face='Lucida Console'>)</font>;
                truth <font color='#5555FF'>=</font> <b>{</b><font color='#979000'>4</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,  <font color='#979000'>4</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>, <font color='#979000'>4</font><b>}</b>;
                <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font><font color='#5555FF'>-</font> truth<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;
            <b>}</b>

            A <font color='#5555FF'>=</font> <font color='#979000'>4</font>;
            A.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            B.<font color='#BB00BB'>device</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <b>{</b>
                <font color='#009900'>// non-aliasing test
</font>                <font color='#0000FF'>auto</font> aA <font color='#5555FF'>=</font> <font color='#BB00BB'>at</font><font face='Lucida Console'>(</font>A,<font color='#979000'>5</font><font face='Lucida Console'>)</font>;
                <font color='#0000FF'>auto</font> aB <font color='#5555FF'>=</font> <font color='#BB00BB'>at</font><font face='Lucida Console'>(</font>B,<font color='#979000'>5</font><font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>memcpy</font><font face='Lucida Console'>(</font>aA, aB<font face='Lucida Console'>)</font>;
                truth <font color='#5555FF'>=</font> <b>{</b><font color='#979000'>4</font>,<font color='#979000'>4</font>,<font color='#979000'>4</font>,<font color='#979000'>4</font>,<font color='#979000'>4</font>,  <font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>, <font color='#979000'>4</font><b>}</b>;
                <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font><font color='#5555FF'>-</font> truth<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;
            <b>}</b>
            <b>{</b>
                <font color='#009900'>// aliasing test
</font>                <font color='#0000FF'>auto</font> aA <font color='#5555FF'>=</font> <font color='#BB00BB'>at</font><font face='Lucida Console'>(</font>A,<font color='#979000'>1</font><font face='Lucida Console'>)</font>;
                <font color='#0000FF'>auto</font> aB <font color='#5555FF'>=</font> <font color='#BB00BB'>at</font><font face='Lucida Console'>(</font>A,<font color='#979000'>6</font><font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>memcpy</font><font face='Lucida Console'>(</font>aA, aB<font face='Lucida Console'>)</font>;
                truth <font color='#5555FF'>=</font> <b>{</b><font color='#979000'>4</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,  <font color='#979000'>4</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>, <font color='#979000'>4</font><b>}</b>;
                <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font><font color='#5555FF'>-</font> truth<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;
            <b>}</b>

<font color='#0000FF'>#endif</font>
        <b>}</b>

        <b>{</b>
            resizable_tensor <font color='#BB00BB'>A</font><font face='Lucida Console'>(</font><font color='#979000'>4</font>,<font color='#979000'>5</font><font face='Lucida Console'>)</font>, <font color='#BB00BB'>B</font><font face='Lucida Console'>(</font><font color='#979000'>4</font><font face='Lucida Console'>)</font>;

            tensor_rand rnd;
            rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font>;
            rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font>;

            <font color='#0000FF'><u>float</u></font> alpha <font color='#5555FF'>=</font> <font color='#979000'>1.4</font>;
            <font color='#0000FF'><u>float</u></font> beta <font color='#5555FF'>=</font> <font color='#979000'>0.5</font>;

            matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font> <font color='#BB00BB'>a</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>, <font color='#BB00BB'>b</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> c <font color='#5555FF'>=</font> <font color='#979000'>0</font>; c <font color='#5555FF'>&lt;</font> a.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>c<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#BB00BB'>set_colm</font><font face='Lucida Console'>(</font>a,c<font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> beta<font color='#5555FF'>*</font><font color='#BB00BB'>colm</font><font face='Lucida Console'>(</font>a,c<font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> alpha<font color='#5555FF'>*</font>b;
            <b>}</b>

            tt::<font color='#BB00BB'>add</font><font face='Lucida Console'>(</font>beta, A, alpha, B<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font><font color='#5555FF'>-</font>a<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font>, <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font><font color='#5555FF'>-</font>a<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

            beta <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> c <font color='#5555FF'>=</font> <font color='#979000'>0</font>; c <font color='#5555FF'>&lt;</font> a.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>c<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#BB00BB'>set_colm</font><font face='Lucida Console'>(</font>a,c<font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> beta<font color='#5555FF'>*</font><font color='#BB00BB'>colm</font><font face='Lucida Console'>(</font>a,c<font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> alpha<font color='#5555FF'>*</font>b;
            <b>}</b>

            tt::<font color='#BB00BB'>add</font><font face='Lucida Console'>(</font>beta, A, alpha, B<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font><font color='#5555FF'>-</font>a<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        <b>{</b>
            resizable_tensor A, B;
            A.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font><font color='#979000'>2</font>,<font color='#979000'>3</font>,<font color='#979000'>4</font>,<font color='#979000'>5</font><font face='Lucida Console'>)</font>;
            B.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font><font color='#979000'>2</font>,<font color='#979000'>3</font>,<font color='#979000'>4</font>,<font color='#979000'>5</font><font face='Lucida Console'>)</font>;

            tensor_rand rnd;
            rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font>;
            rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font>;

            matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font> truth;

            truth <font color='#5555FF'>=</font> <font color='#979000'>2</font><font color='#5555FF'>*</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> <font color='#979000'>3</font><font color='#5555FF'>*</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font>;
            tt::<font color='#BB00BB'>add</font><font face='Lucida Console'>(</font><font color='#979000'>2</font>, A, <font color='#979000'>3</font>, B<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font><font color='#5555FF'>-</font>truth <font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font><font face='Lucida Console'>)</font>;


            rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font>;
            rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font>;
            truth <font color='#5555FF'>=</font> <font color='#979000'>0</font><font color='#5555FF'>*</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> <font color='#979000'>3</font><font color='#5555FF'>*</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font>;
            tt::<font color='#BB00BB'>add</font><font face='Lucida Console'>(</font><font color='#979000'>0</font>, A, <font color='#979000'>3</font>, B<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font><font color='#5555FF'>-</font>truth <font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font><font face='Lucida Console'>)</font>;

            rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font>;
            rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font>;
            truth <font color='#5555FF'>=</font> <font color='#979000'>1</font><font color='#5555FF'>*</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> <font color='#979000'>0</font><font color='#5555FF'>*</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font>;
            tt::<font color='#BB00BB'>add</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>, A, <font color='#979000'>0</font>, B<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font><font color='#5555FF'>-</font>truth <font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font><font face='Lucida Console'>)</font>;


            rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font>;
            rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font>;
            truth <font color='#5555FF'>=</font> <font color='#979000'>0</font><font color='#5555FF'>*</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> <font color='#979000'>0</font><font color='#5555FF'>*</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font>;
            tt::<font color='#BB00BB'>add</font><font face='Lucida Console'>(</font><font color='#979000'>0</font>, A, <font color='#979000'>0</font>, B<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font><font color='#5555FF'>-</font>truth <font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font><font face='Lucida Console'>)</font>;


            B.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,<font color='#979000'>3</font>,<font color='#979000'>4</font>,<font color='#979000'>5</font><font face='Lucida Console'>)</font>;
            rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font>;
            rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font>;
            truth <font color='#5555FF'>=</font> <font color='#979000'>2</font><font color='#5555FF'>*</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> <font color='#979000'>3</font><font color='#5555FF'>*</font><font color='#BB00BB'>join_cols</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font>, <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            tt::<font color='#BB00BB'>add</font><font face='Lucida Console'>(</font><font color='#979000'>2</font>, A, <font color='#979000'>3</font>, B<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font><font color='#5555FF'>-</font>truth <font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>A.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font><font color='#979000'>2</font><font face='Lucida Console'>)</font>;

            B.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>4</font>,<font color='#979000'>5</font><font face='Lucida Console'>)</font>;
            rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font>;
            rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font>;
            matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font> temp <font color='#5555FF'>=</font> <font color='#BB00BB'>join_rows</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font>, <font color='#BB00BB'>join_rows</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font>,<font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            truth <font color='#5555FF'>=</font> <font color='#979000'>2</font><font color='#5555FF'>*</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> <font color='#979000'>3</font><font color='#5555FF'>*</font><font color='#BB00BB'>join_cols</font><font face='Lucida Console'>(</font>temp,temp<font face='Lucida Console'>)</font>;
            tt::<font color='#BB00BB'>add</font><font face='Lucida Console'>(</font><font color='#979000'>2</font>, A, <font color='#979000'>3</font>, B<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font><font color='#5555FF'>-</font>truth <font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font>, <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font><font color='#5555FF'>-</font>truth <font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

            B.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,<font color='#979000'>3</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font>;
            rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font>;
            resizable_tensor <font color='#BB00BB'>AA</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font>, <font color='#BB00BB'>BB</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font>;
            tt::<font color='#BB00BB'>add</font><font face='Lucida Console'>(</font><font color='#979000'>2</font>, A, <font color='#979000'>3</font>, B<font face='Lucida Console'>)</font>;
            cpu::<font color='#BB00BB'>add</font><font face='Lucida Console'>(</font><font color='#979000'>2</font>, AA, <font color='#979000'>3</font>, BB<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>AA<font face='Lucida Console'>)</font> <font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font>, <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>AA<font face='Lucida Console'>)</font> <font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        <b>{</b>
            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            resizable_tensor <font color='#BB00BB'>dest1</font><font face='Lucida Console'>(</font><font color='#979000'>123</font>,<font color='#979000'>456</font><font face='Lucida Console'>)</font>, <font color='#BB00BB'>dest2</font><font face='Lucida Console'>(</font><font color='#979000'>123</font>,<font color='#979000'>456</font><font face='Lucida Console'>)</font>;
            resizable_tensor <font color='#BB00BB'>src1</font><font face='Lucida Console'>(</font><font color='#979000'>123</font>,<font color='#979000'>456</font><font face='Lucida Console'>)</font>, <font color='#BB00BB'>src2</font><font face='Lucida Console'>(</font><font color='#979000'>123</font>,<font color='#979000'>456</font><font face='Lucida Console'>)</font>;

            tt::tensor_rand rnd;

            rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>src1<font face='Lucida Console'>)</font>; tt::<font color='#BB00BB'>affine_transform</font><font face='Lucida Console'>(</font>src1, src1, <font color='#979000'>1</font>, <font color='#979000'>2</font><font face='Lucida Console'>)</font>; src2 <font color='#5555FF'>=</font> src1;  <font color='#009900'>// random in range [2, 3]
</font>            dest1 <font color='#5555FF'>=</font> <font color='#BB00BB'>exp</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>src1<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            tt::<font color='#BB00BB'>exp</font><font face='Lucida Console'>(</font>dest2, src2<font face='Lucida Console'>)</font>;
            tt::<font color='#BB00BB'>exp</font><font face='Lucida Console'>(</font>src2, src2<font face='Lucida Console'>)</font>; <font color='#009900'>// should work in place
</font>            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font>, <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>src2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;

            rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>src1<font face='Lucida Console'>)</font>; tt::<font color='#BB00BB'>affine_transform</font><font face='Lucida Console'>(</font>src1, src1, <font color='#979000'>1</font>, <font color='#979000'>2</font><font face='Lucida Console'>)</font>; src2 <font color='#5555FF'>=</font> src1;  <font color='#009900'>// random in range [2, 3]
</font>            dest1 <font color='#5555FF'>=</font> <font color='#BB00BB'>log</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>src1<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            tt::<font color='#BB00BB'>log</font><font face='Lucida Console'>(</font>dest2, src2<font face='Lucida Console'>)</font>;
            tt::<font color='#BB00BB'>log</font><font face='Lucida Console'>(</font>src2, src2<font face='Lucida Console'>)</font>; <font color='#009900'>// should work in place
</font>            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>src2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;

            rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>src1<font face='Lucida Console'>)</font>; tt::<font color='#BB00BB'>affine_transform</font><font face='Lucida Console'>(</font>src1, src1, <font color='#979000'>1</font>, <font color='#979000'>2</font><font face='Lucida Console'>)</font>; src2 <font color='#5555FF'>=</font> src1;  <font color='#009900'>// random in range [2, 3]
</font>            dest1 <font color='#5555FF'>=</font> <font color='#BB00BB'>log10</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>src1<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            tt::<font color='#BB00BB'>log10</font><font face='Lucida Console'>(</font>dest2, src2<font face='Lucida Console'>)</font>;
            tt::<font color='#BB00BB'>log10</font><font face='Lucida Console'>(</font>src2, src2<font face='Lucida Console'>)</font>; <font color='#009900'>// should work in place
</font>            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>src2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;

        <b>}</b>
    <b>}</b>

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
<font color='#0000FF'>#ifdef</font> DLIB_USE_CUDA

    <font color='#0000FF'><u>void</u></font> <b><a name='test_scale_channels'></a>test_scale_channels</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
    <b>{</b>
        tt::tensor_rand rnd;

        resizable_tensor <font color='#BB00BB'>dest1</font><font face='Lucida Console'>(</font><font color='#979000'>2</font>,<font color='#979000'>3</font>,<font color='#979000'>4</font>,<font color='#979000'>5</font><font face='Lucida Console'>)</font>, dest2;
        rnd.<font color='#BB00BB'>fill_gaussian</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font>;
        dest2 <font color='#5555FF'>=</font> dest1;

        resizable_tensor <font color='#BB00BB'>src</font><font face='Lucida Console'>(</font><font color='#979000'>2</font>,<font color='#979000'>3</font>,<font color='#979000'>4</font>,<font color='#979000'>5</font><font face='Lucida Console'>)</font>;
        resizable_tensor <font color='#BB00BB'>scales</font><font face='Lucida Console'>(</font><font color='#979000'>2</font>,<font color='#979000'>3</font><font face='Lucida Console'>)</font>;
        rnd.<font color='#BB00BB'>fill_gaussian</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;
        rnd.<font color='#BB00BB'>fill_gaussian</font><font face='Lucida Console'>(</font>scales<font face='Lucida Console'>)</font>;


        cpu::<font color='#BB00BB'>scale_channels</font><font face='Lucida Console'>(</font><font color='#979000'>true</font>, dest1, src, scales<font face='Lucida Console'>)</font>;
        cuda::<font color='#BB00BB'>scale_channels</font><font face='Lucida Console'>(</font><font color='#979000'>true</font>, dest2, src, scales<font face='Lucida Console'>)</font>;

        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font><font face='Lucida Console'>)</font>;

        cpu::<font color='#BB00BB'>scale_channels</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, dest1, src, scales<font face='Lucida Console'>)</font>;
        cuda::<font color='#BB00BB'>scale_channels</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, dest2, src, scales<font face='Lucida Console'>)</font>;

        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font><font face='Lucida Console'>)</font>;
    <b>}</b>

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'><u>void</u></font> <b><a name='test_affine_rect'></a>test_affine_rect</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
    <b>{</b>
        dlib::rand rnd;

        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>int</u></font> iter <font color='#5555FF'>=</font> <font color='#979000'>0</font>; iter <font color='#5555FF'>&lt;</font> <font color='#979000'>20</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>iter<font face='Lucida Console'>)</font>
        <b>{</b>

            <font color='#0000FF'><u>long</u></font> nr <font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>+</font> rnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>10</font>;
            <font color='#0000FF'><u>long</u></font> nc <font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>+</font> rnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>10</font>;

            resizable_tensor <font color='#BB00BB'>dest1</font><font face='Lucida Console'>(</font>nr,nc<font face='Lucida Console'>)</font>, <font color='#BB00BB'>dest2</font><font face='Lucida Console'>(</font>nr,nc<font face='Lucida Console'>)</font>, <font color='#BB00BB'>src1</font><font face='Lucida Console'>(</font>nr,nc<font face='Lucida Console'>)</font>, <font color='#BB00BB'>src2</font><font face='Lucida Console'>(</font>nr,nc<font face='Lucida Console'>)</font>, <font color='#BB00BB'>src3</font><font face='Lucida Console'>(</font>nr,nc<font face='Lucida Console'>)</font>;
            matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font> dest3;

            dest1 <font color='#5555FF'>=</font> <font color='#979000'>1</font>;
            dest2 <font color='#5555FF'>=</font> <font color='#979000'>1</font>;
            dest3 <font color='#5555FF'>=</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font>;
            src1 <font color='#5555FF'>=</font> <font color='#979000'>2</font>;
            src2 <font color='#5555FF'>=</font> <font color='#979000'>3</font>;
            src3 <font color='#5555FF'>=</font> <font color='#979000'>4</font>;

            point <font color='#BB00BB'>p1</font><font face='Lucida Console'>(</font>rnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font>nc, rnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font>nr<font face='Lucida Console'>)</font>;
            point <font color='#BB00BB'>p2</font><font face='Lucida Console'>(</font>rnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font>nc, rnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font>nr<font face='Lucida Console'>)</font>;
            rectangle <font color='#BB00BB'>rect</font><font face='Lucida Console'>(</font>p1,p2<font face='Lucida Console'>)</font>;

            cuda::<font color='#BB00BB'>affine_transform</font><font face='Lucida Console'>(</font>rect, dest1, src1, src2, src3, <font color='#979000'>2</font>,<font color='#979000'>3</font>,<font color='#979000'>4</font><font face='Lucida Console'>)</font>;

            cpu::<font color='#BB00BB'>affine_transform</font><font face='Lucida Console'>(</font>rect, dest2, src1, src2, src3, <font color='#979000'>2</font>,<font color='#979000'>3</font>,<font color='#979000'>4</font><font face='Lucida Console'>)</font>;

            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

            <font color='#BB00BB'>set_subm</font><font face='Lucida Console'>(</font>dest3,rect<font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>2</font><font color='#5555FF'>*</font><font color='#BB00BB'>subm</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>src1<font face='Lucida Console'>)</font>,rect<font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> <font color='#979000'>3</font><font color='#5555FF'>*</font><font color='#BB00BB'>subm</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>src2<font face='Lucida Console'>)</font>,rect<font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> <font color='#979000'>4</font><font color='#5555FF'>*</font><font color='#BB00BB'>subm</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>src3<font face='Lucida Console'>)</font>,rect<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>dest3 <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

            dest1 <font color='#5555FF'>=</font> <font color='#979000'>1</font>;
            tt::<font color='#BB00BB'>affine_transform</font><font face='Lucida Console'>(</font>rect, dest1, src1, src2, src3, <font color='#979000'>2</font>,<font color='#979000'>3</font>,<font color='#979000'>4</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>dest3 <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <b>}</b>
    <b>}</b>

    <font color='#0000FF'><u>void</u></font> <b><a name='test_conv'></a>test_conv</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
    <b>{</b>
        cuda::tensor_conv conv1;
        cpu::tensor_conv conv2;

        dlib::rand prnd;
        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>int</u></font> iter <font color='#5555FF'>=</font> <font color='#979000'>0</font>; iter <font color='#5555FF'>&lt;</font> <font color='#979000'>400</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>iter<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

            resizable_tensor <font color='#BB00BB'>data</font><font face='Lucida Console'>(</font>prnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>5</font><font color='#5555FF'>+</font><font color='#979000'>1</font>,
                prnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>5</font><font color='#5555FF'>+</font><font color='#979000'>1</font>,
                prnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>25</font><font color='#5555FF'>+</font><font color='#979000'>1</font>,
                prnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>25</font><font color='#5555FF'>+</font><font color='#979000'>1</font>
            <font face='Lucida Console'>)</font>;
            resizable_tensor <font color='#BB00BB'>filters</font><font face='Lucida Console'>(</font>
                prnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>5</font><font color='#5555FF'>+</font><font color='#979000'>1</font>,
                data.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,
                prnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>6</font><font color='#5555FF'>+</font><font color='#979000'>1</font>,
                prnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>6</font><font color='#5555FF'>+</font><font color='#979000'>1</font> 
            <font face='Lucida Console'>)</font>;

            tt::tensor_rand rnd;
            rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>data<font face='Lucida Console'>)</font>;
            rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>filters<font face='Lucida Console'>)</font>;


            resizable_tensor output1, output2;


            <font color='#0000FF'>const</font> <font color='#0000FF'><u>int</u></font> stride_y <font color='#5555FF'>=</font> prnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>5</font><font color='#5555FF'>+</font><font color='#979000'>1</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>int</u></font> stride_x <font color='#5555FF'>=</font> prnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>5</font><font color='#5555FF'>+</font><font color='#979000'>1</font>;
            <font color='#0000FF'><u>int</u></font> padding_y <font color='#5555FF'>=</font> prnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font face='Lucida Console'>(</font>filters.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font color='#979000'>2</font><font color='#5555FF'>+</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'><u>int</u></font> padding_x <font color='#5555FF'>=</font> prnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font face='Lucida Console'>(</font>filters.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font color='#979000'>2</font><font color='#5555FF'>+</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#5555FF'>!</font><font face='Lucida Console'>(</font>filters.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>=</font> data.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> <font color='#979000'>2</font><font color='#5555FF'>*</font>padding_y<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                padding_y <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font>filters.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>-</font>data.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font><font color='#979000'>1</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font color='#979000'>2</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#5555FF'>!</font><font face='Lucida Console'>(</font>filters.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>=</font> data.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> <font color='#979000'>2</font><font color='#5555FF'>*</font>padding_x<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                padding_x <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font>filters.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>-</font>data.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font><font color='#979000'>1</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font color='#979000'>2</font>;
            conv1.<font color='#BB00BB'>setup</font><font face='Lucida Console'>(</font>data,filters,stride_y,stride_x,padding_y,padding_x<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>conv1</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, output1, data, filters<font face='Lucida Console'>)</font>;
            conv2.<font color='#BB00BB'>setup</font><font face='Lucida Console'>(</font>data,filters,stride_y,stride_x,padding_y,padding_x<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>conv2</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, output2, data, filters<font face='Lucida Console'>)</font>;
            dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>forward error: </font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>output1<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>output2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>output1<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>output2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>3</font>, <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>output1<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>output2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                 <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>\n\t padding_y: </font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> padding_y 
                 <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>\n\t padding_x: </font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> padding_x 
                 <font face='Lucida Console'>)</font>;

            <font color='#BB00BB'>conv1</font><font face='Lucida Console'>(</font><font color='#979000'>true</font>, output1, data, filters<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>conv2</font><font face='Lucida Console'>(</font><font color='#979000'>true</font>, output2, data, filters<font face='Lucida Console'>)</font>;
            dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>forward error: </font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>output1<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>output2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>output1<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>output2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>3</font>, <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>output1<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>output2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                 <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>\n\t padding_y: </font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> padding_y 
                 <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>\n\t padding_x: </font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> padding_x 
                 <font face='Lucida Console'>)</font>;



            resizable_tensor gi, data_gradient1, data_gradient2;
            gi.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>output1<font face='Lucida Console'>)</font>;
            rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>gi<font face='Lucida Console'>)</font>;

            data_gradient1.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>data<font face='Lucida Console'>)</font>;
            data_gradient2.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>data<font face='Lucida Console'>)</font>;
            data_gradient1 <font color='#5555FF'>=</font> <font color='#979000'>1</font>;
            data_gradient2 <font color='#5555FF'>=</font> <font color='#979000'>1</font>;

            conv1.<font color='#BB00BB'>get_gradient_for_data</font><font face='Lucida Console'>(</font><font color='#979000'>true</font>, gi, filters, data_gradient1<font face='Lucida Console'>)</font>;
            conv2.<font color='#BB00BB'>get_gradient_for_data</font><font face='Lucida Console'>(</font><font color='#979000'>true</font>, gi, filters, data_gradient2<font face='Lucida Console'>)</font>;

            dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>data gradient error: </font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>data_gradient1<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>data_gradient2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>data_gradient1<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>data_gradient2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>3</font><font face='Lucida Console'>)</font>;

            conv1.<font color='#BB00BB'>get_gradient_for_data</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, gi, filters, data_gradient1<font face='Lucida Console'>)</font>;
            conv2.<font color='#BB00BB'>get_gradient_for_data</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, gi, filters, data_gradient2<font face='Lucida Console'>)</font>;

            dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>data gradient error: </font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>data_gradient1<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>data_gradient2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>data_gradient1<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>data_gradient2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>3</font><font face='Lucida Console'>)</font>;


            resizable_tensor filter_gradient1, filter_gradient2;
            gi.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>output1<font face='Lucida Console'>)</font>;
            rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>gi<font face='Lucida Console'>)</font>;

            filter_gradient1.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>filters<font face='Lucida Console'>)</font>;
            filter_gradient2.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>filters<font face='Lucida Console'>)</font>;
            filter_gradient1 <font color='#5555FF'>=</font> <font color='#979000'>1</font>;
            filter_gradient2 <font color='#5555FF'>=</font> <font color='#979000'>1</font>;

            conv1.<font color='#BB00BB'>get_gradient_for_filters</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, gi, data, filter_gradient1<font face='Lucida Console'>)</font>;
            conv2.<font color='#BB00BB'>get_gradient_for_filters</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, gi, data, filter_gradient2<font face='Lucida Console'>)</font>;

            dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>filter gradient error: </font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>filter_gradient1<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>filter_gradient2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>filter_gradient1<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>filter_gradient2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>3</font>, <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>filter_gradient1<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>filter_gradient2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;


            conv1.<font color='#BB00BB'>get_gradient_for_filters</font><font face='Lucida Console'>(</font><font color='#979000'>true</font>, gi, data, filter_gradient1<font face='Lucida Console'>)</font>;
            conv2.<font color='#BB00BB'>get_gradient_for_filters</font><font face='Lucida Console'>(</font><font color='#979000'>true</font>, gi, data, filter_gradient2<font face='Lucida Console'>)</font>;

            dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>filter gradient error: </font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>filter_gradient1<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>filter_gradient2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>filter_gradient1<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>filter_gradient2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>2e</font><font color='#5555FF'>-</font><font color='#979000'>3</font>, <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>filter_gradient1<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>filter_gradient2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <b>}</b>
    <b>}</b>

    <font color='#0000FF'><u>void</u></font> <b><a name='compare_adam'></a>compare_adam</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#0000FF'><u>float</u></font> t <font color='#5555FF'>=</font> <font color='#979000'>2</font>;
        tt::tensor_rand rnd;
        resizable_tensor s, m, v, params, params_grad;
        s.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font><font color='#979000'>89</font>,<font color='#979000'>90</font>,<font color='#979000'>60</font>,<font color='#979000'>73</font><font face='Lucida Console'>)</font>;
        m.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>s<font face='Lucida Console'>)</font>;
        v.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>s<font face='Lucida Console'>)</font>;
        params.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>s<font face='Lucida Console'>)</font>;
        params_grad.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>s<font face='Lucida Console'>)</font>;

        rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>s<font face='Lucida Console'>)</font>;
        rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>m<font face='Lucida Console'>)</font>;
        rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>v<font face='Lucida Console'>)</font>;
        rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>params<font face='Lucida Console'>)</font>;
        rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>params_grad<font face='Lucida Console'>)</font>;

        resizable_tensor <font color='#BB00BB'>mm</font><font face='Lucida Console'>(</font>m<font face='Lucida Console'>)</font>, <font color='#BB00BB'>vv</font><font face='Lucida Console'>(</font>v<font face='Lucida Console'>)</font>;
        cpu::<font color='#BB00BB'>compute_adam_update</font><font face='Lucida Console'>(</font><font color='#979000'>0</font>,params.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,s, mm, vv, t, <font color='#979000'>0.01</font>, <font color='#979000'>0.001</font>, <font color='#979000'>0.9</font>, <font color='#979000'>0.99</font>, params, params_grad<font face='Lucida Console'>)</font>;
        matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font> s1 <font color='#5555FF'>=</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>s<font face='Lucida Console'>)</font>;
        
        rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>s<font face='Lucida Console'>)</font>;
        cuda::<font color='#BB00BB'>compute_adam_update</font><font face='Lucida Console'>(</font><font color='#979000'>0</font>,params.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,s, m, v, t, <font color='#979000'>0.01</font>, <font color='#979000'>0.001</font>, <font color='#979000'>0.9</font>, <font color='#979000'>0.99</font>, params, params_grad<font face='Lucida Console'>)</font>;
        matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font> s2 <font color='#5555FF'>=</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>s<font face='Lucida Console'>)</font>;

        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>s1<font color='#5555FF'>-</font>s2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font>, <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>s1<font color='#5555FF'>-</font>s2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>m<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>mm<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font>, <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>m<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>mm<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>v<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>vv<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font>, <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>v<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>vv<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
    <b>}</b>

    <font color='#0000FF'><u>void</u></font> <b><a name='test_multiply_zero_padded'></a>test_multiply_zero_padded</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        dlib::rand rnd;
        tt::tensor_rand trnd;
        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>int</u></font> iter <font color='#5555FF'>=</font> <font color='#979000'>0</font>; iter <font color='#5555FF'>&lt;</font> <font color='#979000'>300</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>iter<font face='Lucida Console'>)</font>
        <b>{</b>
            resizable_tensor <font color='#BB00BB'>dest1</font><font face='Lucida Console'>(</font>rnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>4</font><font color='#5555FF'>+</font><font color='#979000'>1</font>,
                                  rnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>4</font><font color='#5555FF'>+</font><font color='#979000'>1</font>,
                                  rnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>4</font><font color='#5555FF'>+</font><font color='#979000'>1</font>,
                                  rnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>4</font><font color='#5555FF'>+</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            resizable_tensor dest2;
            dest2.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font>;
            resizable_tensor <font color='#BB00BB'>src1</font><font face='Lucida Console'>(</font>rnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>4</font><font color='#5555FF'>+</font><font color='#979000'>1</font>,
                                  rnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>4</font><font color='#5555FF'>+</font><font color='#979000'>1</font>,
                                  rnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>4</font><font color='#5555FF'>+</font><font color='#979000'>1</font>,
                                  rnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>4</font><font color='#5555FF'>+</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            resizable_tensor <font color='#BB00BB'>src2</font><font face='Lucida Console'>(</font>rnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>4</font><font color='#5555FF'>+</font><font color='#979000'>1</font>,
                                  rnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>4</font><font color='#5555FF'>+</font><font color='#979000'>1</font>,
                                  rnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>4</font><font color='#5555FF'>+</font><font color='#979000'>1</font>,
                                  rnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>4</font><font color='#5555FF'>+</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>;

            trnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font>;
            trnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font>;
            trnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>src1<font face='Lucida Console'>)</font>;
            trnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>src2<font face='Lucida Console'>)</font>;
            cpu::<font color='#BB00BB'>multiply_zero_padded</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, dest1, src1, src2<font face='Lucida Console'>)</font>;
            cuda::<font color='#BB00BB'>multiply_zero_padded</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, dest2, src1, src2<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;

            cpu::<font color='#BB00BB'>multiply_zero_padded</font><font face='Lucida Console'>(</font><font color='#979000'>true</font>, dest1, src1, src2<font face='Lucida Console'>)</font>;
            cuda::<font color='#BB00BB'>multiply_zero_padded</font><font face='Lucida Console'>(</font><font color='#979000'>true</font>, dest2, src1, src2<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#009900'>// make sure we have a test for the case where all tensors have the same
</font>        <font color='#009900'>// dimensions.
</font>        resizable_tensor <font color='#BB00BB'>dest1</font><font face='Lucida Console'>(</font><font color='#979000'>3</font>,<font color='#979000'>4</font>,<font color='#979000'>5</font>,<font color='#979000'>6</font><font face='Lucida Console'>)</font>;
        resizable_tensor dest2;
        resizable_tensor src1;
        resizable_tensor src2;
        dest2.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font>;
        src1.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font>;
        src2.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font>;

        trnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font>;
        trnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font>;
        trnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>src1<font face='Lucida Console'>)</font>;
        trnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>src2<font face='Lucida Console'>)</font>;
        cpu::<font color='#BB00BB'>multiply_zero_padded</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, dest1, src1, src2<font face='Lucida Console'>)</font>;
        cuda::<font color='#BB00BB'>multiply_zero_padded</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, dest2, src1, src2<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;

        cpu::<font color='#BB00BB'>multiply_zero_padded</font><font face='Lucida Console'>(</font><font color='#979000'>true</font>, dest1, src1, src2<font face='Lucida Console'>)</font>;
        cuda::<font color='#BB00BB'>multiply_zero_padded</font><font face='Lucida Console'>(</font><font color='#979000'>true</font>, dest2, src1, src2<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;
    <b>}</b>

    <font color='#0000FF'><u>void</u></font> <b><a name='test_add'></a>test_add</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        dlib::rand rnd;
        tt::tensor_rand trnd;
        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>int</u></font> iter <font color='#5555FF'>=</font> <font color='#979000'>0</font>; iter <font color='#5555FF'>&lt;</font> <font color='#979000'>300</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>iter<font face='Lucida Console'>)</font>
        <b>{</b>
            resizable_tensor <font color='#BB00BB'>dest1</font><font face='Lucida Console'>(</font>rnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>4</font><font color='#5555FF'>+</font><font color='#979000'>1</font>,
                                  rnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>4</font><font color='#5555FF'>+</font><font color='#979000'>1</font>,
                                  rnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>4</font><font color='#5555FF'>+</font><font color='#979000'>1</font>,
                                  rnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>4</font><font color='#5555FF'>+</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            resizable_tensor dest2;
            dest2.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font>;
            resizable_tensor <font color='#BB00BB'>src1</font><font face='Lucida Console'>(</font>rnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>4</font><font color='#5555FF'>+</font><font color='#979000'>1</font>,
                                  rnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>4</font><font color='#5555FF'>+</font><font color='#979000'>1</font>,
                                  rnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>4</font><font color='#5555FF'>+</font><font color='#979000'>1</font>,
                                  rnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>4</font><font color='#5555FF'>+</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            resizable_tensor <font color='#BB00BB'>src2</font><font face='Lucida Console'>(</font>rnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>4</font><font color='#5555FF'>+</font><font color='#979000'>1</font>,
                                  rnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>4</font><font color='#5555FF'>+</font><font color='#979000'>1</font>,
                                  rnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>4</font><font color='#5555FF'>+</font><font color='#979000'>1</font>,
                                  rnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>4</font><font color='#5555FF'>+</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>;

            trnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font>;
            trnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font>;
            trnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>src1<font face='Lucida Console'>)</font>;
            trnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>src2<font face='Lucida Console'>)</font>;
            cpu::<font color='#BB00BB'>add</font><font face='Lucida Console'>(</font>dest1, src1, src2<font face='Lucida Console'>)</font>;
            cuda::<font color='#BB00BB'>add</font><font face='Lucida Console'>(</font>dest2, src1, src2<font face='Lucida Console'>)</font>;

            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#009900'>// make sure we have a test for the case where all tensors have the same
</font>        <font color='#009900'>// dimensions.
</font>        resizable_tensor <font color='#BB00BB'>dest1</font><font face='Lucida Console'>(</font><font color='#979000'>3</font>,<font color='#979000'>4</font>,<font color='#979000'>5</font>,<font color='#979000'>6</font><font face='Lucida Console'>)</font>;
        resizable_tensor dest2;
        resizable_tensor src1;
        resizable_tensor src2;
        dest2.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font>;
        src1.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font>;
        src2.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font>;

        trnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font>;
        trnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font>;
        trnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>src1<font face='Lucida Console'>)</font>;
        trnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>src2<font face='Lucida Console'>)</font>;

        cpu::<font color='#BB00BB'>add</font><font face='Lucida Console'>(</font>dest1, src1, src2<font face='Lucida Console'>)</font>;
        cuda::<font color='#BB00BB'>add</font><font face='Lucida Console'>(</font>dest2, src1, src2<font face='Lucida Console'>)</font>;

        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;
    <b>}</b>

    <font color='#0000FF'><u>void</u></font> <b><a name='test_more_ops'></a>test_more_ops</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> nr, <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> nc<font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#0000FF'>using</font> <font color='#0000FF'>namespace</font> dlib::tt;
        <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <font color='#009900'>// We are going to make sure that the CPU implementation of these things matches
</font>        <font color='#009900'>// the CUDA implementation.
</font>
        tensor_rand rnd;

        resizable_tensor <font color='#BB00BB'>dest</font><font face='Lucida Console'>(</font>nr,nc<font face='Lucida Console'>)</font>, <font color='#BB00BB'>src</font><font face='Lucida Console'>(</font>nr,nc<font face='Lucida Console'>)</font>, dest2, src2;
        resizable_tensor <font color='#BB00BB'>srcb</font><font face='Lucida Console'>(</font>nr,nc<font face='Lucida Console'>)</font>, <font color='#BB00BB'>srcc</font><font face='Lucida Console'>(</font>nr,nc<font face='Lucida Console'>)</font>, srcb2, srcc2;


        rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font>;
        rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;
        dest2 <font color='#5555FF'>=</font> dest; src2 <font color='#5555FF'>=</font> src;
        cuda::<font color='#BB00BB'>multiply</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, dest, dest, src<font face='Lucida Console'>)</font>;
        cpu::<font color='#BB00BB'>multiply</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, dest2, dest2, src2<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>equal</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font>,<font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        cuda::<font color='#BB00BB'>multiply</font><font face='Lucida Console'>(</font><font color='#979000'>true</font>, dest, dest, src<font face='Lucida Console'>)</font>;
        cpu::<font color='#BB00BB'>multiply</font><font face='Lucida Console'>(</font><font color='#979000'>true</font>, dest2, dest2, src2<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>equal</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font>,<font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;


        rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font>;
        rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;
        dest2 <font color='#5555FF'>=</font> dest; src2 <font color='#5555FF'>=</font> src;
        cuda::<font color='#BB00BB'>affine_transform</font><font face='Lucida Console'>(</font>dest, src, <font color='#979000'>2</font>, <font color='#979000'>3</font><font face='Lucida Console'>)</font>;
        cpu::<font color='#BB00BB'>affine_transform</font><font face='Lucida Console'>(</font>dest2, src2, <font color='#979000'>2</font>, <font color='#979000'>3</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>equal</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font>,<font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

        rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font>;
        rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;
        rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>srcb<font face='Lucida Console'>)</font>;
        dest2 <font color='#5555FF'>=</font> dest; src2 <font color='#5555FF'>=</font> src; srcb2 <font color='#5555FF'>=</font> srcb;
        cuda::<font color='#BB00BB'>affine_transform</font><font face='Lucida Console'>(</font>dest, src, srcb, <font color='#979000'>2</font>, <font color='#979000'>3</font>, <font color='#979000'>4</font><font face='Lucida Console'>)</font>;
        cpu::<font color='#BB00BB'>affine_transform</font><font face='Lucida Console'>(</font>dest2, src2, srcb2, <font color='#979000'>2</font>, <font color='#979000'>3</font>, <font color='#979000'>4</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>equal</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font>,<font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

        rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font>;
        rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;
        rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>srcb<font face='Lucida Console'>)</font>;
        rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>srcc<font face='Lucida Console'>)</font>;
        dest2 <font color='#5555FF'>=</font> dest; src2 <font color='#5555FF'>=</font> src; srcb2 <font color='#5555FF'>=</font> srcb; srcc2 <font color='#5555FF'>=</font> srcc;
        cuda::<font color='#BB00BB'>affine_transform</font><font face='Lucida Console'>(</font>dest, src, srcb, srcc, <font color='#979000'>2</font>, <font color='#979000'>3</font>, <font color='#979000'>4</font>, <font color='#979000'>5</font><font face='Lucida Console'>)</font>;
        cpu::<font color='#BB00BB'>affine_transform</font><font face='Lucida Console'>(</font>dest2, src2, srcb2, srcc2, <font color='#979000'>2</font>, <font color='#979000'>3</font>, <font color='#979000'>4</font>, <font color='#979000'>5</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>equal</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font>,<font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

        cuda::<font color='#BB00BB'>affine_transform</font><font face='Lucida Console'>(</font>dest, src, srcb, srcc, <font color='#979000'>2</font>, <font color='#979000'>3</font>, <font color='#979000'>4</font>, <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
        cpu::<font color='#BB00BB'>affine_transform</font><font face='Lucida Console'>(</font>dest2, src2, srcb2, srcc2, <font color='#979000'>2</font>, <font color='#979000'>3</font>, <font color='#979000'>4</font>, <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>equal</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font>,<font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

        cuda::<font color='#BB00BB'>affine_transform_range</font><font face='Lucida Console'>(</font><font color='#979000'>0</font>, dest.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, dest, src, srcb, srcc, <font color='#979000'>2</font>, <font color='#979000'>3</font>, <font color='#979000'>4</font><font face='Lucida Console'>)</font>;
        cpu::<font color='#BB00BB'>affine_transform_range</font><font face='Lucida Console'>(</font><font color='#979000'>0</font>, dest2.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, dest2, src2, srcb2, srcc2, <font color='#979000'>2</font>, <font color='#979000'>3</font>, <font color='#979000'>4</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>equal</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font>,<font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

        <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#979000'>3</font> <font color='#5555FF'>&lt;</font> dest.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
        <b>{</b>
            dest <font color='#5555FF'>=</font> <font color='#979000'>999</font>;
            dest2 <font color='#5555FF'>=</font> <font color='#979000'>999</font>;
            cuda::<font color='#BB00BB'>affine_transform_range</font><font face='Lucida Console'>(</font><font color='#979000'>3</font>, dest.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#979000'>1</font>, dest, src, srcb, srcc, <font color='#979000'>2</font>, <font color='#979000'>3</font>, <font color='#979000'>4</font><font face='Lucida Console'>)</font>;
            cpu::<font color='#BB00BB'>affine_transform_range</font><font face='Lucida Console'>(</font><font color='#979000'>3</font>, dest2.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#979000'>1</font>, dest2, src2, srcb2, srcc2, <font color='#979000'>2</font>, <font color='#979000'>3</font>, <font color='#979000'>4</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>equal</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font>,<font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

            cuda::<font color='#BB00BB'>affine_transform_range</font><font face='Lucida Console'>(</font>dest.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, dest.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, dest, src, srcb, srcc, <font color='#979000'>2</font>, <font color='#979000'>3</font>, <font color='#979000'>4</font><font face='Lucida Console'>)</font>;
            cpu::<font color='#BB00BB'>affine_transform_range</font><font face='Lucida Console'>(</font>dest2.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, dest2.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, dest2, src2, srcb2, srcc2, <font color='#979000'>2</font>, <font color='#979000'>3</font>, <font color='#979000'>4</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>equal</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font>,<font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <b>}</b>


        rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font>;
        rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;
        rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>srcb<font face='Lucida Console'>)</font>;
        rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>srcc<font face='Lucida Console'>)</font>;
        dest2 <font color='#5555FF'>=</font> dest; src2 <font color='#5555FF'>=</font> src; srcb2 <font color='#5555FF'>=</font> srcb; srcc2 <font color='#5555FF'>=</font> srcc;
        cuda::<font color='#BB00BB'>affine_transform</font><font face='Lucida Console'>(</font>dest, src, srcb, srcc<font face='Lucida Console'>)</font>;
        cpu::<font color='#BB00BB'>affine_transform</font><font face='Lucida Console'>(</font>dest2, src2, srcb2, srcc2<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>equal</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font>,<font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <font color='#009900'>// now exercise code path where the A/B tensors have num_samples()==1
</font>        srcb.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,nc<font face='Lucida Console'>)</font>;
        srcc.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,nc<font face='Lucida Console'>)</font>;
        rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font>;
        rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;
        rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>srcb<font face='Lucida Console'>)</font>;
        rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>srcc<font face='Lucida Console'>)</font>;
        dest2 <font color='#5555FF'>=</font> dest; src2 <font color='#5555FF'>=</font> src; srcb2 <font color='#5555FF'>=</font> srcb; srcc2 <font color='#5555FF'>=</font> srcc;
        cuda::<font color='#BB00BB'>affine_transform</font><font face='Lucida Console'>(</font>dest, src, srcb, srcc<font face='Lucida Console'>)</font>;
        cpu::<font color='#BB00BB'>affine_transform</font><font face='Lucida Console'>(</font>dest2, src2, srcb2, srcc2<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>equal</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font>,<font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;


        rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;
        src2 <font color='#5555FF'>=</font> src;
        cuda::<font color='#BB00BB'>threshold</font><font face='Lucida Console'>(</font>src, <font color='#979000'>0.5</font><font face='Lucida Console'>)</font>;
        cpu::<font color='#BB00BB'>threshold</font><font face='Lucida Console'>(</font>src2, <font color='#979000'>0.5</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>equal</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>,<font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>src2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

        <b>{</b>
            resizable_tensor <font color='#BB00BB'>dest</font><font face='Lucida Console'>(</font><font color='#979000'>3</font>,<font color='#979000'>4</font><font face='Lucida Console'>)</font>;
            resizable_tensor A, B;
            A <font color='#5555FF'>=</font> dest;
            B <font color='#5555FF'>=</font> dest;

            rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font>;
            rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font>;
            rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font>;

            dest.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,<font color='#979000'>4</font><font face='Lucida Console'>)</font>;

            cuda::<font color='#BB00BB'>multiply</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, dest, A, B<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>sum_rows</font><font face='Lucida Console'>(</font><font color='#BB00BB'>pointwise_multiply</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font>,<font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font>, <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>sum_rows</font><font face='Lucida Console'>(</font><font color='#BB00BB'>pointwise_multiply</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font>,<font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>; 

            A.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,<font color='#979000'>4</font><font face='Lucida Console'>)</font>;
            rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font>;
            matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font> AA <font color='#5555FF'>=</font> <font color='#BB00BB'>join_cols</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font>,<font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>; AA <font color='#5555FF'>=</font> <font color='#BB00BB'>join_cols</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font>,AA<font face='Lucida Console'>)</font>;

            cuda::<font color='#BB00BB'>multiply</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, dest, A, B<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>sum_rows</font><font face='Lucida Console'>(</font><font color='#BB00BB'>pointwise_multiply</font><font face='Lucida Console'>(</font>AA,<font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font><font face='Lucida Console'>)</font>; 

            cuda::<font color='#BB00BB'>multiply</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, dest, B, A<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>sum_rows</font><font face='Lucida Console'>(</font><font color='#BB00BB'>pointwise_multiply</font><font face='Lucida Console'>(</font>AA,<font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font><font face='Lucida Console'>)</font>; 
            matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font> prevdest <font color='#5555FF'>=</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font>;
            cuda::<font color='#BB00BB'>multiply</font><font face='Lucida Console'>(</font><font color='#979000'>true</font>, dest, B, A<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font><font color='#5555FF'>-</font>prevdest<font color='#5555FF'>-</font><font color='#BB00BB'>sum_rows</font><font face='Lucida Console'>(</font><font color='#BB00BB'>pointwise_multiply</font><font face='Lucida Console'>(</font>AA,<font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font><font face='Lucida Console'>)</font>; 

            dest.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font><font color='#979000'>3</font>,<font color='#979000'>4</font><font face='Lucida Console'>)</font>;
            cuda::<font color='#BB00BB'>multiply</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, dest, B, A<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>pointwise_multiply</font><font face='Lucida Console'>(</font>AA,<font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font><font face='Lucida Console'>)</font>; 
            prevdest <font color='#5555FF'>=</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font>;
            cuda::<font color='#BB00BB'>multiply</font><font face='Lucida Console'>(</font><font color='#979000'>true</font>, dest, B, A<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font><font color='#5555FF'>-</font>prevdest<font color='#5555FF'>-</font><font color='#BB00BB'>pointwise_multiply</font><font face='Lucida Console'>(</font>AA,<font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font><font face='Lucida Console'>)</font>; 

            cuda::<font color='#BB00BB'>multiply</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, dest, A, B<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>pointwise_multiply</font><font face='Lucida Console'>(</font>AA,<font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font><font face='Lucida Console'>)</font>; 
        <b>}</b>

        <b>{</b>
            resizable_tensor invnorms1, invnorms2;
            resizable_tensor <font color='#BB00BB'>data</font><font face='Lucida Console'>(</font><font color='#979000'>4</font>,<font color='#979000'>5</font><font face='Lucida Console'>)</font>, out1, out2;
            rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>data<font face='Lucida Console'>)</font>;

            <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> eps <font color='#5555FF'>=</font> <font color='#979000'>0.1</font>;

            invnorms2 <font color='#5555FF'>=</font> <font color='#BB00BB'>reciprocal</font><font face='Lucida Console'>(</font><font color='#BB00BB'>sqrt</font><font face='Lucida Console'>(</font><font color='#BB00BB'>sum_cols</font><font face='Lucida Console'>(</font><font color='#BB00BB'>squared</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>data<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> eps<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            tt::<font color='#BB00BB'>inverse_norms</font><font face='Lucida Console'>(</font>invnorms1, data, eps<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>invnorms1<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>invnorms2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font><font face='Lucida Console'>)</font>;

            out1.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>data<font face='Lucida Console'>)</font>;
            tt::<font color='#BB00BB'>scale_rows</font><font face='Lucida Console'>(</font>out1, data, invnorms1<font face='Lucida Console'>)</font>;
            out2 <font color='#5555FF'>=</font> <font color='#BB00BB'>scale_rows</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>data<font face='Lucida Console'>)</font>, <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>invnorms1<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>out1<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>out2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        <b>{</b>
            resizable_tensor <font color='#BB00BB'>a</font><font face='Lucida Console'>(</font><font color='#979000'>123</font>,<font color='#979000'>432</font><font face='Lucida Console'>)</font>, <font color='#BB00BB'>b</font><font face='Lucida Console'>(</font><font color='#979000'>123</font>,<font color='#979000'>432</font><font face='Lucida Console'>)</font>;
            rnd.<font color='#BB00BB'>fill_gaussian</font><font face='Lucida Console'>(</font>a<font face='Lucida Console'>)</font>;
            rnd.<font color='#BB00BB'>fill_gaussian</font><font face='Lucida Console'>(</font>b<font face='Lucida Console'>)</font>;

            resizable_tensor out;
            <font color='#BB00BB'>dot_prods</font><font face='Lucida Console'>(</font>out, a,b<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font> truth <font color='#5555FF'>=</font> <font color='#BB00BB'>sum_cols</font><font face='Lucida Console'>(</font><font color='#BB00BB'>pointwise_multiply</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>a<font face='Lucida Console'>)</font>, <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>b<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>out<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> truth<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>4</font><font face='Lucida Console'>)</font>;
            out <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>out<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> truth<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&gt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>2</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>dot_prods</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, out, a,b<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>out<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> truth<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>4</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>dot_prods</font><font face='Lucida Console'>(</font><font color='#979000'>true</font>, out, a,b<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>out<font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font color='#979000'>2</font> <font color='#5555FF'>-</font> truth<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>4</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>out<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> truth<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&gt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>2</font><font face='Lucida Console'>)</font>;
        <b>}</b>
    <b>}</b>

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'><u>void</u></font> <b><a name='compare_bn_gpu_and_cpu'></a>compare_bn_gpu_and_cpu</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        resizable_tensor dest, dest2;
        resizable_tensor means, means2;
        resizable_tensor invstds, invstds2;
        resizable_tensor running_means, running_means2;
        resizable_tensor running_variances, running_variances2;
        resizable_tensor <font color='#BB00BB'>src</font><font face='Lucida Console'>(</font><font color='#979000'>64</font>,<font color='#979000'>20</font>,<font color='#979000'>100</font>,<font color='#979000'>100</font><font face='Lucida Console'>)</font>;
        resizable_tensor <font color='#BB00BB'>gamma</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,<font color='#979000'>20</font>,<font color='#979000'>100</font>,<font color='#979000'>100</font><font face='Lucida Console'>)</font>;
        resizable_tensor <font color='#BB00BB'>beta</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,<font color='#979000'>20</font>,<font color='#979000'>100</font>,<font color='#979000'>100</font><font face='Lucida Console'>)</font>;
        gamma <font color='#5555FF'>=</font> <font color='#979000'>2</font>;
        beta <font color='#5555FF'>=</font> <font color='#979000'>3</font>;
        tt::tensor_rand rnd;
        rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;


        cpu::<font color='#BB00BB'>batch_normalize</font><font face='Lucida Console'>(</font>DEFAULT_BATCH_NORM_EPS,dest, means, invstds, <font color='#979000'>1</font>, running_means, running_variances, src, gamma, beta<font face='Lucida Console'>)</font>;
        cuda::<font color='#BB00BB'>batch_normalize</font><font face='Lucida Console'>(</font>DEFAULT_BATCH_NORM_EPS,dest2,means2,invstds2, <font color='#979000'>1</font>, running_means2, running_variances2, src, gamma, beta<font face='Lucida Console'>)</font>;

        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>dest error:    </font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>means error:   </font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>means<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>means2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>invstds error: </font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>invstds<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>invstds2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>running_means error:   </font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>running_means<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>running_means2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>running_variances error: </font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>running_variances<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>running_variances2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>4</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>means<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>means2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>4</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>invstds<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>invstds2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>4</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>running_means<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>running_means2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>4</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>running_variances<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>running_variances2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>4</font>,
            <font color='#BB00BB'>mean</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>running_variances<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> 
            <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>\n</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>mean</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>running_variances2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
            <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>\n</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>running_variances<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>running_variances2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
            <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>\n</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>mean</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>running_variances<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>running_variances2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
            <font face='Lucida Console'>)</font>;


        <font color='#009900'>// now check that the gradients match as well
</font>        resizable_tensor gradient_input;
        resizable_tensor src_grad, gamma_grad, beta_grad;
        resizable_tensor src_grad2, gamma_grad2, beta_grad2;
        gradient_input.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font>;
        src_grad.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>; src_grad <font color='#5555FF'>=</font> <font color='#979000'>0</font>; src_grad2 <font color='#5555FF'>=</font> src_grad;
        gamma_grad.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>gamma<font face='Lucida Console'>)</font>; gamma_grad <font color='#5555FF'>=</font> <font color='#979000'>0</font>; gamma_grad2 <font color='#5555FF'>=</font> gamma_grad;
        beta_grad.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>beta<font face='Lucida Console'>)</font>; beta_grad <font color='#5555FF'>=</font> <font color='#979000'>0</font>; beta_grad2 <font color='#5555FF'>=</font> beta_grad;
        rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>gradient_input<font face='Lucida Console'>)</font>;


        cpu::<font color='#BB00BB'>batch_normalize_gradient</font><font face='Lucida Console'>(</font>DEFAULT_BATCH_NORM_EPS,gradient_input, means, invstds, src, gamma, src_grad, gamma_grad, beta_grad<font face='Lucida Console'>)</font>;
        cuda::<font color='#BB00BB'>batch_normalize_gradient</font><font face='Lucida Console'>(</font>DEFAULT_BATCH_NORM_EPS,gradient_input, means, invstds, src, gamma, src_grad2, gamma_grad2, beta_grad2<font face='Lucida Console'>)</font>;

        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>src_grad error:   </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>src_grad<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>src_grad2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>gamma_grad error: </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>gamma_grad<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>gamma_grad2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>beta_grad error:  </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>beta_grad<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>beta_grad2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>src_grad<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>src_grad2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>4</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>gamma_grad<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>gamma_grad2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>4</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>beta_grad<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>beta_grad2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>4</font><font face='Lucida Console'>)</font>;
    <b>}</b>

    <font color='#0000FF'><u>void</u></font> <b><a name='compare_bn_conv_gpu_and_cpu'></a>compare_bn_conv_gpu_and_cpu</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        resizable_tensor dest, dest2;
        resizable_tensor means, means2;
        resizable_tensor invstds, invstds2;
        resizable_tensor running_means, running_means2;
        resizable_tensor running_variances, running_variances2;
        resizable_tensor <font color='#BB00BB'>src</font><font face='Lucida Console'>(</font><font color='#979000'>2</font>,<font color='#979000'>8</font>,<font color='#979000'>10</font>,<font color='#979000'>9</font><font face='Lucida Console'>)</font>;
        resizable_tensor <font color='#BB00BB'>gamma</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,<font color='#979000'>8</font><font face='Lucida Console'>)</font>;
        resizable_tensor <font color='#BB00BB'>beta</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,<font color='#979000'>8</font><font face='Lucida Console'>)</font>;
        gamma <font color='#5555FF'>=</font> <font color='#979000'>2</font>;
        beta <font color='#5555FF'>=</font> <font color='#979000'>3</font>;
        tt::tensor_rand rnd;
        rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;

        cpu::<font color='#BB00BB'>batch_normalize_conv</font><font face='Lucida Console'>(</font>DEFAULT_BATCH_NORM_EPS,dest,means,invstds,<font color='#979000'>1</font>,running_means,running_variances, src, gamma, beta<font face='Lucida Console'>)</font>;
        cuda::<font color='#BB00BB'>batch_normalize_conv</font><font face='Lucida Console'>(</font>DEFAULT_BATCH_NORM_EPS,dest2,means2,invstds2,<font color='#979000'>1</font>,running_means2,running_variances2, src, gamma, beta<font face='Lucida Console'>)</font>;

        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>dest error:    </font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>means error:   </font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>means<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>means2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>invstds error: </font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>invstds<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>invstds2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>running_means error:   </font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>running_means<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>running_means2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>running_variances error: </font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>running_variances<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>running_variances2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>4</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>means<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>means2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>4</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>invstds<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>invstds2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>4</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>running_means<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>running_means2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>4</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>running_variances<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>running_variances2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>4</font><font face='Lucida Console'>)</font>;

        resizable_tensor gradient_input;
        resizable_tensor src_grad, gamma_grad, beta_grad;
        resizable_tensor src_grad2, gamma_grad2, beta_grad2;
        gradient_input.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font>;
        src_grad.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>; src_grad <font color='#5555FF'>=</font> <font color='#979000'>0</font>; src_grad2 <font color='#5555FF'>=</font> src_grad;
        gamma_grad.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>gamma<font face='Lucida Console'>)</font>; gamma_grad <font color='#5555FF'>=</font> <font color='#979000'>0</font>; gamma_grad2 <font color='#5555FF'>=</font> gamma_grad;
        beta_grad.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>beta<font face='Lucida Console'>)</font>; beta_grad <font color='#5555FF'>=</font> <font color='#979000'>0</font>; beta_grad2 <font color='#5555FF'>=</font> beta_grad;
        rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>gradient_input<font face='Lucida Console'>)</font>;


        cpu::<font color='#BB00BB'>batch_normalize_conv_gradient</font><font face='Lucida Console'>(</font>DEFAULT_BATCH_NORM_EPS,gradient_input, means, invstds, src, gamma, src_grad, gamma_grad, beta_grad<font face='Lucida Console'>)</font>;
        cuda::<font color='#BB00BB'>batch_normalize_conv_gradient</font><font face='Lucida Console'>(</font>DEFAULT_BATCH_NORM_EPS,gradient_input, means, invstds, src, gamma, src_grad2, gamma_grad2, beta_grad2<font face='Lucida Console'>)</font>;

        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>src_grad error:   </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>src_grad<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>src_grad2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>gamma_grad error: </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>gamma_grad<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>gamma_grad2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>beta_grad error:  </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>beta_grad<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>beta_grad2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>src_grad<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>src_grad2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>4</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>gamma_grad<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>gamma_grad2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>4</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>beta_grad<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>beta_grad2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>4</font><font face='Lucida Console'>)</font>;
    <b>}</b>


    <font color='#0000FF'><u>void</u></font> <b><a name='test_more_ops2'></a>test_more_ops2</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
    <b>{</b>
        dlib::rand rnd;
        tt::tensor_rand trand;

        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>int</u></font> iter <font color='#5555FF'>=</font> <font color='#979000'>0</font>; iter <font color='#5555FF'>&lt;</font> <font color='#979000'>100</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>iter<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            resizable_tensor dest1, dest2, src1, src2;
            src1.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font>rnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>30</font><font color='#5555FF'>+</font><font color='#979000'>1</font>,
                rnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>30</font><font color='#5555FF'>+</font><font color='#979000'>1</font>,
                rnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>30</font><font color='#5555FF'>+</font><font color='#979000'>1</font>,
                rnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>30</font><font color='#5555FF'>+</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            dest1.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>src1<font face='Lucida Console'>)</font>;
            dest2.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>src1<font face='Lucida Console'>)</font>;
            src2.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,src1.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font><font face='Lucida Console'>)</font>;

            trand.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font>;
            trand.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font>;
            trand.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>src1<font face='Lucida Console'>)</font>;
            trand.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>src2<font face='Lucida Console'>)</font>;

            cpu::<font color='#BB00BB'>multiply_conv</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, dest1, src1, src2<font face='Lucida Console'>)</font>;
            cuda::<font color='#BB00BB'>multiply_conv</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, dest2, src1, src2<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;
            cpu::<font color='#BB00BB'>multiply_conv</font><font face='Lucida Console'>(</font><font color='#979000'>true</font>, dest1, src1, src2<font face='Lucida Console'>)</font>;
            cuda::<font color='#BB00BB'>multiply_conv</font><font face='Lucida Console'>(</font><font color='#979000'>true</font>, dest2, src1, src2<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;


            <font color='#009900'>// now try it using the other mode of multiply_conv
</font>            src2.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>src1<font face='Lucida Console'>)</font>;
            dest1.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,src1.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            dest2.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,src1.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            trand.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font>;
            trand.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font>;
            trand.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>src1<font face='Lucida Console'>)</font>;
            trand.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>src2<font face='Lucida Console'>)</font>;
            cpu::<font color='#BB00BB'>multiply_conv</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, dest1, src1, src2<font face='Lucida Console'>)</font>;
            cuda::<font color='#BB00BB'>multiply_conv</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, dest2, src1, src2<font face='Lucida Console'>)</font>;
            <font color='#0000FF'><u>float</u></font> scale <font color='#5555FF'>=</font> <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'><u>float</u></font> scalem <font color='#5555FF'>=</font> <font color='#BB00BB'>mean</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font>scale <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>4</font> , <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font>scale<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mean</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font>scalem <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font> , <font color='#BB00BB'>mean</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font>scalem<font face='Lucida Console'>)</font>;
            matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font> prevd2 <font color='#5555FF'>=</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font>;
            cpu::<font color='#BB00BB'>multiply_conv</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, dest1, src1, src2<font face='Lucida Console'>)</font>;
            cuda::<font color='#BB00BB'>multiply_conv</font><font face='Lucida Console'>(</font><font color='#979000'>true</font>, dest2, src1, src2<font face='Lucida Console'>)</font>;
            scale <font color='#5555FF'>=</font> <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            scalem <font color='#5555FF'>=</font> <font color='#BB00BB'>mean</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font color='#5555FF'>+</font>prevd2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font>scale <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>4</font> , <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font color='#5555FF'>+</font>prevd2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font>scale<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mean</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font color='#5555FF'>+</font>prevd2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font>scalem <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font> , <font color='#BB00BB'>mean</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font color='#5555FF'>+</font>prevd2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font>scalem<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>int</u></font> iter <font color='#5555FF'>=</font> <font color='#979000'>0</font>; iter <font color='#5555FF'>&lt;</font> <font color='#979000'>100</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>iter<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            resizable_tensor dest1, dest2, src, A, B;
            src.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font>rnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>30</font><font color='#5555FF'>+</font><font color='#979000'>1</font>,
                rnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>30</font><font color='#5555FF'>+</font><font color='#979000'>1</font>,
                rnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>30</font><font color='#5555FF'>+</font><font color='#979000'>1</font>,
                rnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>30</font><font color='#5555FF'>+</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            dest1.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;
            dest2.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;
            A.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            B.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font><font face='Lucida Console'>)</font>;

            trand.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font>;
            trand.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font>;
            trand.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;
            trand.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font>;
            trand.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>B<font face='Lucida Console'>)</font>;

            cpu::<font color='#BB00BB'>affine_transform_conv</font><font face='Lucida Console'>(</font>dest1, src, A, B<font face='Lucida Console'>)</font>;
            cuda::<font color='#BB00BB'>affine_transform_conv</font><font face='Lucida Console'>(</font>dest2, src, A, B<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>int</u></font> iter <font color='#5555FF'>=</font> <font color='#979000'>0</font>; iter <font color='#5555FF'>&lt;</font> <font color='#979000'>100</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>iter<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            resizable_tensor dest1, dest2, g;
            g.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font>rnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>30</font><font color='#5555FF'>+</font><font color='#979000'>1</font>,
                rnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>30</font><font color='#5555FF'>+</font><font color='#979000'>1</font>,
                rnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>30</font><font color='#5555FF'>+</font><font color='#979000'>1</font>,
                rnd.<font color='#BB00BB'>get_random_32bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font><font color='#979000'>30</font><font color='#5555FF'>+</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            dest1.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,g.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            dest2.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,g.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font><font face='Lucida Console'>)</font>;

            trand.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font>;
            trand.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font>;
            trand.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>g<font face='Lucida Console'>)</font>;

            cpu::<font color='#BB00BB'>assign_conv_bias_gradient</font><font face='Lucida Console'>(</font>dest1, g<font face='Lucida Console'>)</font>;
            cuda::<font color='#BB00BB'>assign_conv_bias_gradient</font><font face='Lucida Console'>(</font>dest2, g<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> scale <font color='#5555FF'>=</font> <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> scalem <font color='#5555FF'>=</font> <font color='#BB00BB'>mean</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font>scale <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>4</font> , <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font>scale<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mean</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font>scalem <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font> , <font color='#BB00BB'>mean</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest1<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>dest2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font>scalem<font face='Lucida Console'>)</font>;
        <b>}</b>

    <b>}</b>

<font color='#0000FF'>#endif</font> <font color='#009900'>// DLIB_USE_CUDA
</font>
<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'><u>void</u></font> <b><a name='test_max_pool'></a>test_max_pool</b><font face='Lucida Console'>(</font>
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>int</u></font> window_height,
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>int</u></font> window_width,
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>int</u></font> stride_y,
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>int</u></font> stride_x,
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>int</u></font> padding_y,
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>int</u></font> padding_x
    <font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        resizable_tensor A, B, gradient_input;
        A.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font><font color='#979000'>4</font>,<font color='#979000'>5</font>,<font color='#979000'>16</font>,<font color='#979000'>7</font><font face='Lucida Console'>)</font>;
        B.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font>;
        gradient_input.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font>;

        tt::tensor_rand rnd;
        rnd.<font color='#BB00BB'>fill_gaussian</font><font face='Lucida Console'>(</font>A,<font color='#979000'>0</font>,<font color='#979000'>1</font><font face='Lucida Console'>)</font>;
        rnd.<font color='#BB00BB'>fill_gaussian</font><font face='Lucida Console'>(</font>B,<font color='#979000'>0</font>,<font color='#979000'>1</font><font face='Lucida Console'>)</font>;
        rnd.<font color='#BB00BB'>fill_gaussian</font><font face='Lucida Console'>(</font>gradient_input,<font color='#979000'>0</font>,<font color='#979000'>1</font><font face='Lucida Console'>)</font>;


        tt::pooling mp;

        mp.<font color='#BB00BB'>setup_max_pooling</font><font face='Lucida Console'>(</font>window_height,window_width,stride_y,stride_x,padding_y,padding_x<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>mp</font><font face='Lucida Console'>(</font>A, B<font face='Lucida Console'>)</font>;

        <font color='#009900'>// make sure max pooling does what it's spec says it should.
</font>        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font> A.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> B.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font> A.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> B.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font> A.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font color='#5555FF'>+</font><font face='Lucida Console'>(</font>B.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font><font color='#979000'>2</font><font color='#5555FF'>*</font>padding_y<font color='#5555FF'>-</font>window_height<font face='Lucida Console'>)</font><font color='#5555FF'>/</font>stride_y<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font> A.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font color='#5555FF'>+</font><font face='Lucida Console'>(</font>B.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font><font color='#979000'>2</font><font color='#5555FF'>*</font>padding_x<font color='#5555FF'>-</font>window_width<font face='Lucida Console'>)</font><font color='#5555FF'>/</font>stride_x<font face='Lucida Console'>)</font>;

        <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> x_offset <font color='#5555FF'>=</font> window_width<font color='#5555FF'>/</font><font color='#979000'>2</font> <font color='#5555FF'>-</font> padding_x;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> y_offset <font color='#5555FF'>=</font> window_height<font color='#5555FF'>/</font><font color='#979000'>2</font> <font color='#5555FF'>-</font> padding_y;
        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> s <font color='#5555FF'>=</font> <font color='#979000'>0</font>; s <font color='#5555FF'>&lt;</font> A.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>s<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'>&lt;</font> A.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> r <font color='#5555FF'>=</font> <font color='#979000'>0</font>; r <font color='#5555FF'>&lt;</font> A.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>r<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> c <font color='#5555FF'>=</font> <font color='#979000'>0</font>; c <font color='#5555FF'>&lt;</font> A.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>c<font face='Lucida Console'>)</font>
                    <b>{</b>
                        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font><font color='#BB00BB'>image_plane</font><font face='Lucida Console'>(</font>A,s,k<font face='Lucida Console'>)</font><font face='Lucida Console'>(</font>r,c<font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>subm_clipped</font><font face='Lucida Console'>(</font><font color='#BB00BB'>image_plane</font><font face='Lucida Console'>(</font>B,s,k<font face='Lucida Console'>)</font>,
                                    <font color='#BB00BB'>centered_rect</font><font face='Lucida Console'>(</font>c<font color='#5555FF'>*</font>stride_x<font color='#5555FF'>+</font>x_offset,
                                                  r<font color='#5555FF'>*</font>stride_y<font color='#5555FF'>+</font>y_offset,
                                                  window_width,
                                                  window_height<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>, 
                                                  "<font color='#CC0000'>padding: </font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> padding_x <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>  </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> padding_y 
                                                  <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> window size: </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> window_width <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> window_height 
                                                  <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> stride: </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> stride_x <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> stride_y
                                                  <font face='Lucida Console'>)</font>;
                    <b>}</b>
                <b>}</b>
            <b>}</b>
        <b>}</b>
    <b>}</b>

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'><u>void</u></font> <b><a name='test_avg_pool'></a>test_avg_pool</b><font face='Lucida Console'>(</font>
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>int</u></font> window_height,
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>int</u></font> window_width,
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>int</u></font> stride_y,
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>int</u></font> stride_x,
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>int</u></font> padding_y,
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>int</u></font> padding_x
    <font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        resizable_tensor A, B, gradient_input;
        A.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font><font color='#979000'>4</font>,<font color='#979000'>5</font>,<font color='#979000'>16</font>,<font color='#979000'>7</font><font face='Lucida Console'>)</font>;
        B.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font>;
        gradient_input.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>A<font face='Lucida Console'>)</font>;

        tt::tensor_rand rnd;
        rnd.<font color='#BB00BB'>fill_gaussian</font><font face='Lucida Console'>(</font>A,<font color='#979000'>0</font>,<font color='#979000'>1</font><font face='Lucida Console'>)</font>;
        rnd.<font color='#BB00BB'>fill_gaussian</font><font face='Lucida Console'>(</font>B,<font color='#979000'>0</font>,<font color='#979000'>1</font><font face='Lucida Console'>)</font>;
        rnd.<font color='#BB00BB'>fill_gaussian</font><font face='Lucida Console'>(</font>gradient_input,<font color='#979000'>0</font>,<font color='#979000'>1</font><font face='Lucida Console'>)</font>;


        tt::pooling mp;

        mp.<font color='#BB00BB'>setup_avg_pooling</font><font face='Lucida Console'>(</font>window_height,window_width,stride_y,stride_x,padding_y,padding_x<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>mp</font><font face='Lucida Console'>(</font>A, B<font face='Lucida Console'>)</font>;

        <font color='#009900'>// make sure avg pooling does what it's spec says it should.
</font>        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font> A.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> B.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font> A.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> B.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font> A.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font color='#5555FF'>+</font><font face='Lucida Console'>(</font>B.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font><font color='#979000'>2</font><font color='#5555FF'>*</font>padding_y<font color='#5555FF'>-</font>window_height<font face='Lucida Console'>)</font><font color='#5555FF'>/</font>stride_y<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font> A.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font color='#5555FF'>+</font><font face='Lucida Console'>(</font>B.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font><font color='#979000'>2</font><font color='#5555FF'>*</font>padding_x<font color='#5555FF'>-</font>window_width<font face='Lucida Console'>)</font><font color='#5555FF'>/</font>stride_x<font face='Lucida Console'>)</font>;

        <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> x_offset <font color='#5555FF'>=</font> window_width<font color='#5555FF'>/</font><font color='#979000'>2</font> <font color='#5555FF'>-</font> padding_x;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> y_offset <font color='#5555FF'>=</font> window_height<font color='#5555FF'>/</font><font color='#979000'>2</font> <font color='#5555FF'>-</font> padding_y;
        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> s <font color='#5555FF'>=</font> <font color='#979000'>0</font>; s <font color='#5555FF'>&lt;</font> A.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>s<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'>&lt;</font> A.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> r <font color='#5555FF'>=</font> <font color='#979000'>0</font>; r <font color='#5555FF'>&lt;</font> A.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>r<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> c <font color='#5555FF'>=</font> <font color='#979000'>0</font>; c <font color='#5555FF'>&lt;</font> A.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>c<font face='Lucida Console'>)</font>
                    <b>{</b>
                        <font color='#0000FF'><u>float</u></font> expected <font color='#5555FF'>=</font> <font color='#BB00BB'>mean</font><font face='Lucida Console'>(</font><font color='#BB00BB'>subm_clipped</font><font face='Lucida Console'>(</font><font color='#BB00BB'>image_plane</font><font face='Lucida Console'>(</font>B,s,k<font face='Lucida Console'>)</font>,
                                            <font color='#BB00BB'>centered_rect</font><font face='Lucida Console'>(</font>c<font color='#5555FF'>*</font>stride_x<font color='#5555FF'>+</font>x_offset,
                                                        r<font color='#5555FF'>*</font>stride_y<font color='#5555FF'>+</font>y_offset,
                                                        window_width,
                                                        window_height<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                        <font color='#0000FF'><u>float</u></font> err <font color='#5555FF'>=</font> <font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>image_plane</font><font face='Lucida Console'>(</font>A,s,k<font face='Lucida Console'>)</font><font face='Lucida Console'>(</font>r,c<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> expected<font face='Lucida Console'>)</font>;
                        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>err <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font>, err <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>  </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> expected <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>  </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>image_plane</font><font face='Lucida Console'>(</font>A,s,k<font face='Lucida Console'>)</font><font face='Lucida Console'>(</font>r,c<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                    <b>}</b>
                <b>}</b>
            <b>}</b>
        <b>}</b>
    <b>}</b>

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'><u>void</u></font> <b><a name='test_layers'></a>test_layers</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
    <b>{</b>
        <b>{</b>
            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            reorg_<font color='#5555FF'>&lt;</font><font color='#979000'>2</font>,<font color='#979000'>2</font><font color='#5555FF'>&gt;</font> l;
            <font color='#0000FF'>auto</font> res <font color='#5555FF'>=</font> <font color='#BB00BB'>test_layer</font><font face='Lucida Console'>(</font>l<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>res, res<font face='Lucida Console'>)</font>;
        <b>}</b>
        <b>{</b>
            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            extract_<font color='#5555FF'>&lt;</font><font color='#979000'>0</font>,<font color='#979000'>2</font>,<font color='#979000'>2</font>,<font color='#979000'>2</font><font color='#5555FF'>&gt;</font> l;
            <font color='#0000FF'>auto</font> res <font color='#5555FF'>=</font> <font color='#BB00BB'>test_layer</font><font face='Lucida Console'>(</font>l<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>res, res<font face='Lucida Console'>)</font>;
        <b>}</b>
        <b>{</b>
            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            extract_<font color='#5555FF'>&lt;</font><font color='#979000'>3</font>,<font color='#979000'>2</font>,<font color='#979000'>1</font>,<font color='#979000'>2</font><font color='#5555FF'>&gt;</font> l;
            <font color='#0000FF'>auto</font> res <font color='#5555FF'>=</font> <font color='#BB00BB'>test_layer</font><font face='Lucida Console'>(</font>l<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>res, res<font face='Lucida Console'>)</font>;
        <b>}</b>
        <b>{</b>
            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            extract_<font color='#5555FF'>&lt;</font><font color='#979000'>0</font>,<font color='#979000'>2</font>,<font color='#979000'>1</font>,<font color='#979000'>2</font><font color='#5555FF'>&gt;</font> l;
            <font color='#0000FF'>auto</font> res <font color='#5555FF'>=</font> <font color='#BB00BB'>test_layer</font><font face='Lucida Console'>(</font>l<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>res, res<font face='Lucida Console'>)</font>;
        <b>}</b>
        <b>{</b>
            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            upsample_<font color='#5555FF'>&lt;</font><font color='#979000'>1</font>,<font color='#979000'>1</font><font color='#5555FF'>&gt;</font> l;
            <font color='#0000FF'>auto</font> res <font color='#5555FF'>=</font> <font color='#BB00BB'>test_layer</font><font face='Lucida Console'>(</font>l<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>res, res<font face='Lucida Console'>)</font>;
        <b>}</b>
        <b>{</b>
            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            upsample_<font color='#5555FF'>&lt;</font><font color='#979000'>2</font>,<font color='#979000'>1</font><font color='#5555FF'>&gt;</font> l;
            <font color='#0000FF'>auto</font> res <font color='#5555FF'>=</font> <font color='#BB00BB'>test_layer</font><font face='Lucida Console'>(</font>l<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>res, res<font face='Lucida Console'>)</font>;
        <b>}</b>
        <b>{</b>
            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            upsample_<font color='#5555FF'>&lt;</font><font color='#979000'>2</font>,<font color='#979000'>2</font><font color='#5555FF'>&gt;</font> l;
            <font color='#0000FF'>auto</font> res <font color='#5555FF'>=</font> <font color='#BB00BB'>test_layer</font><font face='Lucida Console'>(</font>l<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>res, res<font face='Lucida Console'>)</font>;
        <b>}</b>
        <b>{</b>
            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            upsample_<font color='#5555FF'>&lt;</font><font color='#979000'>3</font>,<font color='#979000'>3</font><font color='#5555FF'>&gt;</font> l;
            <font color='#0000FF'>auto</font> res <font color='#5555FF'>=</font> <font color='#BB00BB'>test_layer</font><font face='Lucida Console'>(</font>l<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>res, res<font face='Lucida Console'>)</font>;
        <b>}</b>
        <b>{</b>
            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            resize_to_<font color='#5555FF'>&lt;</font><font color='#979000'>1</font>,<font color='#979000'>1</font><font color='#5555FF'>&gt;</font> l;
            <font color='#0000FF'>auto</font> res <font color='#5555FF'>=</font> <font color='#BB00BB'>test_layer</font><font face='Lucida Console'>(</font>l<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>res, res<font face='Lucida Console'>)</font>;
        <b>}</b>
        <b>{</b>
            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            resize_to_<font color='#5555FF'>&lt;</font><font color='#979000'>2</font>,<font color='#979000'>1</font><font color='#5555FF'>&gt;</font> l;
            <font color='#0000FF'>auto</font> res <font color='#5555FF'>=</font> <font color='#BB00BB'>test_layer</font><font face='Lucida Console'>(</font>l<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>res, res<font face='Lucida Console'>)</font>;
        <b>}</b>
        <b>{</b>
            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            resize_to_<font color='#5555FF'>&lt;</font><font color='#979000'>2</font>,<font color='#979000'>2</font><font color='#5555FF'>&gt;</font> l;
            <font color='#0000FF'>auto</font> res <font color='#5555FF'>=</font> <font color='#BB00BB'>test_layer</font><font face='Lucida Console'>(</font>l<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>res, res<font face='Lucida Console'>)</font>;
        <b>}</b>
        <b>{</b>
            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            l2normalize_ l;
            <font color='#0000FF'>auto</font> res <font color='#5555FF'>=</font> <font color='#BB00BB'>test_layer</font><font face='Lucida Console'>(</font>l<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>res, res<font face='Lucida Console'>)</font>;
        <b>}</b>
        <b>{</b>
            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            multiply_ l;
            <font color='#0000FF'>auto</font> res <font color='#5555FF'>=</font> <font color='#BB00BB'>test_layer</font><font face='Lucida Console'>(</font>l<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>res, res<font face='Lucida Console'>)</font>;
        <b>}</b>
        <b>{</b>
            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            max_pool_<font color='#5555FF'>&lt;</font><font color='#979000'>3</font>,<font color='#979000'>3</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font><font color='#5555FF'>&gt;</font> l;
            <font color='#0000FF'>auto</font> res <font color='#5555FF'>=</font> <font color='#BB00BB'>test_layer</font><font face='Lucida Console'>(</font>l<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>res, res<font face='Lucida Console'>)</font>;
        <b>}</b>
        <b>{</b>
            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            avg_pool_<font color='#5555FF'>&lt;</font><font color='#979000'>3</font>,<font color='#979000'>3</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font><font color='#5555FF'>&gt;</font> l;
            <font color='#0000FF'>auto</font> res <font color='#5555FF'>=</font> <font color='#BB00BB'>test_layer</font><font face='Lucida Console'>(</font>l<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>res, res<font face='Lucida Console'>)</font>;
        <b>}</b>
        <b>{</b>
            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            affine_ <font color='#BB00BB'>l</font><font face='Lucida Console'>(</font>CONV_MODE<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font> res <font color='#5555FF'>=</font> <font color='#BB00BB'>test_layer</font><font face='Lucida Console'>(</font>l<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>res, res<font face='Lucida Console'>)</font>;
        <b>}</b>
        <b>{</b>
            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            affine_ <font color='#BB00BB'>l</font><font face='Lucida Console'>(</font>FC_MODE<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font> res <font color='#5555FF'>=</font> <font color='#BB00BB'>test_layer</font><font face='Lucida Console'>(</font>l<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>res, res<font face='Lucida Console'>)</font>;
        <b>}</b>
        <b>{</b>
            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            bn_<font color='#5555FF'>&lt;</font>CONV_MODE<font color='#5555FF'>&gt;</font> l;
            <font color='#0000FF'>auto</font> res <font color='#5555FF'>=</font> <font color='#BB00BB'>test_layer</font><font face='Lucida Console'>(</font>l<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>res, res<font face='Lucida Console'>)</font>;
        <b>}</b>
        <b>{</b>
            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            bn_<font color='#5555FF'>&lt;</font>FC_MODE<font color='#5555FF'>&gt;</font> l;
            <font color='#0000FF'>auto</font> res <font color='#5555FF'>=</font> <font color='#BB00BB'>test_layer</font><font face='Lucida Console'>(</font>l<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>res, res<font face='Lucida Console'>)</font>;
        <b>}</b>
        <b>{</b>
            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            layer_norm_ l;
            <font color='#0000FF'>auto</font> res <font color='#5555FF'>=</font> <font color='#BB00BB'>test_layer</font><font face='Lucida Console'>(</font>l<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>res, res<font face='Lucida Console'>)</font>;
        <b>}</b>
        <b>{</b>
            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            cont_<font color='#5555FF'>&lt;</font><font color='#979000'>3</font>,<font color='#979000'>3</font>,<font color='#979000'>3</font>,<font color='#979000'>2</font>,<font color='#979000'>2</font>,<font color='#979000'>0</font>,<font color='#979000'>0</font><font color='#5555FF'>&gt;</font> l;
            <font color='#0000FF'>auto</font> res <font color='#5555FF'>=</font> <font color='#BB00BB'>test_layer</font><font face='Lucida Console'>(</font>l<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>res, res<font face='Lucida Console'>)</font>;
        <b>}</b>
        <b>{</b>
            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            cont_<font color='#5555FF'>&lt;</font><font color='#979000'>3</font>,<font color='#979000'>3</font>,<font color='#979000'>3</font>,<font color='#979000'>2</font>,<font color='#979000'>2</font><font color='#5555FF'>&gt;</font> l;
            <font color='#0000FF'>auto</font> res <font color='#5555FF'>=</font> <font color='#BB00BB'>test_layer</font><font face='Lucida Console'>(</font>l<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>res, res<font face='Lucida Console'>)</font>;
        <b>}</b>
        <b>{</b>
            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            cont_<font color='#5555FF'>&lt;</font><font color='#979000'>3</font>,<font color='#979000'>3</font>,<font color='#979000'>3</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font><font color='#5555FF'>&gt;</font> l;
            <font color='#0000FF'>auto</font> res <font color='#5555FF'>=</font> <font color='#BB00BB'>test_layer</font><font face='Lucida Console'>(</font>l<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>res, res<font face='Lucida Console'>)</font>;
        <b>}</b>
        <b>{</b>
            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            cont_<font color='#5555FF'>&lt;</font><font color='#979000'>3</font>,<font color='#979000'>3</font>,<font color='#979000'>3</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>0</font>,<font color='#979000'>0</font><font color='#5555FF'>&gt;</font> l;
            <font color='#0000FF'>auto</font> res <font color='#5555FF'>=</font> <font color='#BB00BB'>test_layer</font><font face='Lucida Console'>(</font>l<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>res, res<font face='Lucida Console'>)</font>;
        <b>}</b>
        <b>{</b>
            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            cont_<font color='#5555FF'>&lt;</font><font color='#979000'>3</font>,<font color='#979000'>2</font>,<font color='#979000'>2</font>,<font color='#979000'>2</font>,<font color='#979000'>2</font><font color='#5555FF'>&gt;</font> l;
            <font color='#0000FF'>auto</font> res <font color='#5555FF'>=</font> <font color='#BB00BB'>test_layer</font><font face='Lucida Console'>(</font>l<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>res, res<font face='Lucida Console'>)</font>;
        <b>}</b>
        <b>{</b>
            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            con_<font color='#5555FF'>&lt;</font><font color='#979000'>3</font>,<font color='#979000'>2</font>,<font color='#979000'>2</font>,<font color='#979000'>2</font>,<font color='#979000'>2</font><font color='#5555FF'>&gt;</font> l;
            <font color='#0000FF'>auto</font> res <font color='#5555FF'>=</font> <font color='#BB00BB'>test_layer</font><font face='Lucida Console'>(</font>l<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>res, res<font face='Lucida Console'>)</font>;
        <b>}</b>
        <b>{</b>
            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            con_<font color='#5555FF'>&lt;</font><font color='#979000'>3</font>,<font color='#979000'>3</font>,<font color='#979000'>3</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font><font color='#5555FF'>&gt;</font>l;
            <font color='#0000FF'>auto</font> res <font color='#5555FF'>=</font> <font color='#BB00BB'>test_layer</font><font face='Lucida Console'>(</font>l<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>res, res<font face='Lucida Console'>)</font>;
        <b>}</b>
        <b>{</b>
            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            con_<font color='#5555FF'>&lt;</font><font color='#979000'>3</font>,<font color='#979000'>3</font>,<font color='#979000'>2</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font><font color='#5555FF'>&gt;</font> l;
            <font color='#0000FF'>auto</font> res <font color='#5555FF'>=</font> <font color='#BB00BB'>test_layer</font><font face='Lucida Console'>(</font>l<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>res, res<font face='Lucida Console'>)</font>;
        <b>}</b>
        <b>{</b>
            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            con_<font color='#5555FF'>&lt;</font><font color='#979000'>2</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font><font color='#5555FF'>&gt;</font> l;
            <font color='#0000FF'>auto</font> res <font color='#5555FF'>=</font> <font color='#BB00BB'>test_layer</font><font face='Lucida Console'>(</font>l<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>res, res<font face='Lucida Console'>)</font>;
        <b>}</b>
        <b>{</b>
            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            con_<font color='#5555FF'>&lt;</font><font color='#979000'>3</font>,<font color='#979000'>0</font>,<font color='#979000'>2</font>,<font color='#979000'>2</font>,<font color='#979000'>2</font><font color='#5555FF'>&gt;</font> l;
            <font color='#0000FF'>auto</font> res <font color='#5555FF'>=</font> <font color='#BB00BB'>test_layer</font><font face='Lucida Console'>(</font>l<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>res, res<font face='Lucida Console'>)</font>;
        <b>}</b>
        <b>{</b>
            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            con_<font color='#5555FF'>&lt;</font><font color='#979000'>3</font>,<font color='#979000'>2</font>,<font color='#979000'>0</font>,<font color='#979000'>2</font>,<font color='#979000'>2</font><font color='#5555FF'>&gt;</font> l;
            <font color='#0000FF'>auto</font> res <font color='#5555FF'>=</font> <font color='#BB00BB'>test_layer</font><font face='Lucida Console'>(</font>l<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>res, res<font face='Lucida Console'>)</font>;
        <b>}</b>
        <b>{</b>
            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            con_<font color='#5555FF'>&lt;</font><font color='#979000'>3</font>,<font color='#979000'>0</font>,<font color='#979000'>0</font>,<font color='#979000'>2</font>,<font color='#979000'>2</font><font color='#5555FF'>&gt;</font> l;
            <font color='#0000FF'>auto</font> res <font color='#5555FF'>=</font> <font color='#BB00BB'>test_layer</font><font face='Lucida Console'>(</font>l<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>res, res<font face='Lucida Console'>)</font>;
        <b>}</b>
        <b>{</b>
            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            fc_<font color='#5555FF'>&lt;</font><font color='#979000'>1</font>,FC_HAS_BIAS<font color='#5555FF'>&gt;</font> l;
            <font color='#0000FF'>auto</font> res <font color='#5555FF'>=</font> <font color='#BB00BB'>test_layer</font><font face='Lucida Console'>(</font>l<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>res, res<font face='Lucida Console'>)</font>;
        <b>}</b>
        <b>{</b>
            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            fc_<font color='#5555FF'>&lt;</font><font color='#979000'>5</font>,FC_HAS_BIAS<font color='#5555FF'>&gt;</font> l;
            <font color='#0000FF'>auto</font> res <font color='#5555FF'>=</font> <font color='#BB00BB'>test_layer</font><font face='Lucida Console'>(</font>l<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>res, res<font face='Lucida Console'>)</font>;
        <b>}</b>
        <b>{</b>
            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            fc_<font color='#5555FF'>&lt;</font><font color='#979000'>4</font>,FC_NO_BIAS<font color='#5555FF'>&gt;</font> l;
            <font color='#0000FF'>auto</font> res <font color='#5555FF'>=</font> <font color='#BB00BB'>test_layer</font><font face='Lucida Console'>(</font>l<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>res, res<font face='Lucida Console'>)</font>;
        <b>}</b>
        <b>{</b>
            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            relu_ l;
            <font color='#0000FF'>auto</font> res <font color='#5555FF'>=</font> <font color='#BB00BB'>test_layer</font><font face='Lucida Console'>(</font>l<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>res, res<font face='Lucida Console'>)</font>;
        <b>}</b>
        <b>{</b>
            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            prelu_ l;
            <font color='#0000FF'>auto</font> res <font color='#5555FF'>=</font> <font color='#BB00BB'>test_layer</font><font face='Lucida Console'>(</font>l<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>res, res<font face='Lucida Console'>)</font>;
        <b>}</b>
        <b>{</b>
            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            leaky_relu_ l;
            <font color='#0000FF'>auto</font> res <font color='#5555FF'>=</font> <font color='#BB00BB'>test_layer</font><font face='Lucida Console'>(</font>l<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>res, res<font face='Lucida Console'>)</font>;
        <b>}</b>
        <b>{</b>
            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            sig_ l;
            <font color='#0000FF'>auto</font> res <font color='#5555FF'>=</font> <font color='#BB00BB'>test_layer</font><font face='Lucida Console'>(</font>l<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>res, res<font face='Lucida Console'>)</font>;
        <b>}</b>
        <b>{</b>
            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            mish_ l;
            <font color='#0000FF'>auto</font> res <font color='#5555FF'>=</font> <font color='#BB00BB'>test_layer</font><font face='Lucida Console'>(</font>l<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>res, res<font face='Lucida Console'>)</font>;
        <b>}</b>
        <b>{</b>
            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            htan_ l;
            <font color='#0000FF'>auto</font> res <font color='#5555FF'>=</font> <font color='#BB00BB'>test_layer</font><font face='Lucida Console'>(</font>l<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>res, res<font face='Lucida Console'>)</font>;
        <b>}</b>
        <b>{</b>
            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            clipped_relu_ l;
            <font color='#0000FF'>auto</font> res <font color='#5555FF'>=</font> <font color='#BB00BB'>test_layer</font><font face='Lucida Console'>(</font>l<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>res, res<font face='Lucida Console'>)</font>;
        <b>}</b>
        <b>{</b>
            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            elu_ l;
            <font color='#0000FF'>auto</font> res <font color='#5555FF'>=</font> <font color='#BB00BB'>test_layer</font><font face='Lucida Console'>(</font>l<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>res, res<font face='Lucida Console'>)</font>;
        <b>}</b>
        <b>{</b>
            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            gelu_ l;
            <font color='#0000FF'>auto</font> res <font color='#5555FF'>=</font> <font color='#BB00BB'>test_layer</font><font face='Lucida Console'>(</font>l<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>res, res<font face='Lucida Console'>)</font>;
        <b>}</b>
        <b>{</b>
            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            smelu_ l;
            <font color='#0000FF'>auto</font> res <font color='#5555FF'>=</font> <font color='#BB00BB'>test_layer</font><font face='Lucida Console'>(</font>l<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>res, res<font face='Lucida Console'>)</font>;
        <b>}</b>
        <b>{</b>
            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            silu_ l;
            <font color='#0000FF'>auto</font> res <font color='#5555FF'>=</font> <font color='#BB00BB'>test_layer</font><font face='Lucida Console'>(</font>l<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>res, res<font face='Lucida Console'>)</font>;
        <b>}</b>
        <b>{</b>
            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            softmax_ l;
            <font color='#0000FF'>auto</font> res <font color='#5555FF'>=</font> <font color='#BB00BB'>test_layer</font><font face='Lucida Console'>(</font>l<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>res, res<font face='Lucida Console'>)</font>;
        <b>}</b>
        <b>{</b>
            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            softmax_all_ l;
            <font color='#0000FF'>auto</font> res <font color='#5555FF'>=</font> <font color='#BB00BB'>test_layer</font><font face='Lucida Console'>(</font>l<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>res, res<font face='Lucida Console'>)</font>;
        <b>}</b>
    <b>}</b>

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> n, <font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> rcon <font color='#5555FF'>=</font> max_pool<font color='#5555FF'>&lt;</font><font color='#979000'>2</font>,<font color='#979000'>2</font>,<font color='#979000'>2</font>,<font color='#979000'>2</font>,relu<font color='#5555FF'>&lt;</font>bn_con<font color='#5555FF'>&lt;</font>con<font color='#5555FF'>&lt;</font>n,<font color='#979000'>5</font>,<font color='#979000'>5</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,SUBNET<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> n, <font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> rfc <font color='#5555FF'>=</font> relu<font color='#5555FF'>&lt;</font>bn_fc<font color='#5555FF'>&lt;</font>fc<font color='#5555FF'>&lt;</font>n,SUBNET<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>;

    <font color='#0000FF'><u>void</u></font> <b><a name='test_tagging'></a>test_tagging</b><font face='Lucida Console'>(</font>
    <font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#0000FF'>typedef</font> loss_multiclass_log<font color='#5555FF'>&lt;</font>rfc<font color='#5555FF'>&lt;</font><font color='#979000'>10</font>,skip1<font color='#5555FF'>&lt;</font>rfc<font color='#5555FF'>&lt;</font><font color='#979000'>84</font>,rfc<font color='#5555FF'>&lt;</font><font color='#979000'>120</font>,tag1<font color='#5555FF'>&lt;</font>rcon<font color='#5555FF'>&lt;</font><font color='#979000'>16</font>,rcon<font color='#5555FF'>&lt;</font><font color='#979000'>6</font>,input<font color='#5555FF'>&lt;</font>matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>char</u></font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font> net_type;

        net_type net;
        net_type <font color='#BB00BB'>net2</font><font face='Lucida Console'>(</font><font color='#BB00BB'>num_fc_outputs</font><font face='Lucida Console'>(</font><font color='#979000'>4</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>layer<font color='#5555FF'>&lt;</font>tag1<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font>.num_computational_layers <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>8</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>layer<font color='#5555FF'>&lt;</font>skip1<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font>.num_computational_layers <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>8</font><font color='#5555FF'>+</font><font color='#979000'>3</font><font color='#5555FF'>+</font><font color='#979000'>3</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>layer<font color='#5555FF'>&lt;</font>tag1<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font>.num_layers <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>10</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>layer<font color='#5555FF'>&lt;</font>skip1<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font>.num_layers <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>10</font><font color='#5555FF'>+</font><font color='#979000'>3</font><font color='#5555FF'>+</font><font color='#979000'>3</font><font color='#5555FF'>+</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#5555FF'>&amp;</font>layer<font color='#5555FF'>&lt;</font>skip1<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#5555FF'>&amp;</font>layer<font color='#5555FF'>&lt;</font>tag1<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#5555FF'>&amp;</font>layer<font color='#5555FF'>&lt;</font>skip1<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>!</font><font color='#5555FF'>=</font> <font color='#5555FF'>&amp;</font>layer<font color='#5555FF'>&lt;</font>tag1<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font>.<font color='#BB00BB'>subnet</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>subnet</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>net.<font color='#BB00BB'>subnet</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>subnet</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>subnet</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>layer_details</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_num_outputs</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>10</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>net2.<font color='#BB00BB'>subnet</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>subnet</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>subnet</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>layer_details</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_num_outputs</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>4</font><font face='Lucida Console'>)</font>;
    <b>}</b>

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
        <font color='#0000FF'><u>int</u></font> N, 
        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='BN'></a>BN</b>, 
        <font color='#0000FF'><u>int</u></font> stride, 
        <font color='#0000FF'>typename</font> SUBNET
        <font color='#5555FF'>&gt;</font> 
    <font color='#0000FF'>using</font> block  <font color='#5555FF'>=</font> BN<font color='#5555FF'>&lt;</font>con<font color='#5555FF'>&lt;</font>N,<font color='#979000'>3</font>,<font color='#979000'>3</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,relu<font color='#5555FF'>&lt;</font>BN<font color='#5555FF'>&lt;</font>con<font color='#5555FF'>&lt;</font>N,<font color='#979000'>3</font>,<font color='#979000'>3</font>,stride,stride,SUBNET<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'><u>int</u></font>,<font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font><font color='#0000FF'>class</font>,<font color='#0000FF'><u>int</u></font>,<font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='block'></a>block</b>, 
        <font color='#0000FF'><u>int</u></font> N, 
        <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font><font color='#0000FF'>class</font> <b><a name='BN'></a>BN</b>, 
        <font color='#0000FF'>typename</font> SUBNET
        <font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> residual <font color='#5555FF'>=</font> add_prev1<font color='#5555FF'>&lt;</font>block<font color='#5555FF'>&lt;</font>N,BN,<font color='#979000'>1</font>,tag1<font color='#5555FF'>&lt;</font>SUBNET<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'><u>int</u></font>,<font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font><font color='#0000FF'>class</font>,<font color='#0000FF'><u>int</u></font>,<font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='block'></a>block</b>, 
        <font color='#0000FF'><u>int</u></font> N, 
        <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font><font color='#0000FF'>class</font> <b><a name='BN'></a>BN</b>, 
        <font color='#0000FF'>typename</font> SUBNET
        <font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> residual_down <font color='#5555FF'>=</font> add_prev2<font color='#5555FF'>&lt;</font>avg_pool<font color='#5555FF'>&lt;</font><font color='#979000'>2</font>,<font color='#979000'>2</font>,<font color='#979000'>2</font>,<font color='#979000'>2</font>,skip1<font color='#5555FF'>&lt;</font>tag2<font color='#5555FF'>&lt;</font>block<font color='#5555FF'>&lt;</font>N,BN,<font color='#979000'>2</font>,tag1<font color='#5555FF'>&lt;</font>SUBNET<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>;


    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> res       <font color='#5555FF'>=</font> relu<font color='#5555FF'>&lt;</font>residual<font color='#5555FF'>&lt;</font>block,<font color='#979000'>8</font>,bn_con,SUBNET<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> ares      <font color='#5555FF'>=</font> relu<font color='#5555FF'>&lt;</font>residual<font color='#5555FF'>&lt;</font>block,<font color='#979000'>8</font>,affine,SUBNET<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> res_down  <font color='#5555FF'>=</font> relu<font color='#5555FF'>&lt;</font>residual_down<font color='#5555FF'>&lt;</font>block,<font color='#979000'>8</font>,bn_con,SUBNET<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> ares_down <font color='#5555FF'>=</font> relu<font color='#5555FF'>&lt;</font>residual_down<font color='#5555FF'>&lt;</font>block,<font color='#979000'>8</font>,affine,SUBNET<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> 
    <font color='#0000FF'>using</font> pres  <font color='#5555FF'>=</font> prelu<font color='#5555FF'>&lt;</font>add_prev1<font color='#5555FF'>&lt;</font>bn_con<font color='#5555FF'>&lt;</font>con<font color='#5555FF'>&lt;</font><font color='#979000'>8</font>,<font color='#979000'>3</font>,<font color='#979000'>3</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,prelu<font color='#5555FF'>&lt;</font>bn_con<font color='#5555FF'>&lt;</font>con<font color='#5555FF'>&lt;</font><font color='#979000'>8</font>,<font color='#979000'>3</font>,<font color='#979000'>3</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,tag1<font color='#5555FF'>&lt;</font>SUBNET<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>;

    <font color='#0000FF'><u>void</u></font> <b><a name='test_visit_functions'></a>test_visit_functions</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#0000FF'>using</font> net_type2 <font color='#5555FF'>=</font> loss_multiclass_log<font color='#5555FF'>&lt;</font>fc<font color='#5555FF'>&lt;</font><font color='#979000'>10</font>,
            avg_pool_everything<font color='#5555FF'>&lt;</font>
            pres<font color='#5555FF'>&lt;</font>res<font color='#5555FF'>&lt;</font>res<font color='#5555FF'>&lt;</font>res_down<font color='#5555FF'>&lt;</font> <font color='#009900'>// 2 prelu layers here
</font>            tag4<font color='#5555FF'>&lt;</font>repeat<font color='#5555FF'>&lt;</font><font color='#979000'>9</font>,pres,    <font color='#009900'>// 9 groups, each containing 2 prelu layers  
</font>            res_down<font color='#5555FF'>&lt;</font>
            leaky_relu<font color='#5555FF'>&lt;</font>res<font color='#5555FF'>&lt;</font>
            input<font color='#5555FF'>&lt;</font>matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>char</u></font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>
            <font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>;

        net_type2 pnet;
        <font color='#0000FF'>const</font> net_type2<font color='#5555FF'>&amp;</font> const_pnet <font color='#5555FF'>=</font> pnet;

        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>pnet.num_layers <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>132</font>, pnet.num_layers<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>pnet.num_computational_layers <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>110</font>, pnet.num_computational_layers<font face='Lucida Console'>)</font>;

        <b>{</b>
            std::vector<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>bool</u></font><font color='#5555FF'>&gt;</font> <font color='#BB00BB'>hit</font><font face='Lucida Console'>(</font>pnet.num_computational_layers, <font color='#979000'>false</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'><u>size_t</u></font> count <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            <font color='#BB00BB'>visit_layer_parameter_gradients</font><font face='Lucida Console'>(</font>pnet, [<font color='#5555FF'>&amp;</font>]<font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i, tensor<font color='#5555FF'>&amp;</font> <font face='Lucida Console'>)</font><b>{</b>hit[i] <font color='#5555FF'>=</font> <font color='#979000'>true</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>count; <b>}</b><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'>auto</font> x : hit<font face='Lucida Console'>)</font>
                <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>count <font color='#5555FF'>=</font><font color='#5555FF'>=</font> pnet.num_computational_layers<font face='Lucida Console'>)</font>;
        <b>}</b>
        <b>{</b>
            std::vector<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>bool</u></font><font color='#5555FF'>&gt;</font> <font color='#BB00BB'>hit</font><font face='Lucida Console'>(</font>pnet.num_computational_layers, <font color='#979000'>false</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'><u>size_t</u></font> count <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            <font color='#BB00BB'>visit_layer_parameter_gradients</font><font face='Lucida Console'>(</font>const_pnet, [<font color='#5555FF'>&amp;</font>]<font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i, <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <font face='Lucida Console'>)</font><b>{</b>hit[i] <font color='#5555FF'>=</font> <font color='#979000'>true</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>count; <b>}</b><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'>auto</font> x : hit<font face='Lucida Console'>)</font>
                <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>count <font color='#5555FF'>=</font><font color='#5555FF'>=</font> pnet.num_computational_layers<font face='Lucida Console'>)</font>;
        <b>}</b>

        <b>{</b>
            <font color='#0000FF'><u>size_t</u></font> count <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            std::vector<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>bool</u></font><font color='#5555FF'>&gt;</font> <font color='#BB00BB'>hit2</font><font face='Lucida Console'>(</font>pnet.num_computational_layers, <font color='#979000'>false</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>visit_layer_parameters</font><font face='Lucida Console'>(</font>pnet, [<font color='#5555FF'>&amp;</font>]<font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i, tensor<font color='#5555FF'>&amp;</font> <font face='Lucida Console'>)</font><b>{</b>hit2[i] <font color='#5555FF'>=</font> <font color='#979000'>true</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>count; <b>}</b><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'>auto</font> x : hit2<font face='Lucida Console'>)</font>
                <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>count <font color='#5555FF'>=</font><font color='#5555FF'>=</font> pnet.num_computational_layers<font face='Lucida Console'>)</font>;
        <b>}</b>
        <b>{</b>
            <font color='#0000FF'><u>size_t</u></font> count <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            std::vector<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>bool</u></font><font color='#5555FF'>&gt;</font> <font color='#BB00BB'>hit2</font><font face='Lucida Console'>(</font>pnet.num_computational_layers, <font color='#979000'>false</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>visit_layer_parameters</font><font face='Lucida Console'>(</font>const_pnet, [<font color='#5555FF'>&amp;</font>]<font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i, <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <font face='Lucida Console'>)</font><b>{</b>hit2[i] <font color='#5555FF'>=</font> <font color='#979000'>true</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>count; <b>}</b><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'>auto</font> x : hit2<font face='Lucida Console'>)</font>
                <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>count <font color='#5555FF'>=</font><font color='#5555FF'>=</font> pnet.num_computational_layers<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'><u>int</u></font> num_relus <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        <font color='#BB00BB'>visit_computational_layers</font><font face='Lucida Console'>(</font>pnet, [<font color='#5555FF'>&amp;</font>num_relus]<font face='Lucida Console'>(</font>relu_<font color='#5555FF'>&amp;</font><font face='Lucida Console'>)</font> <b>{</b> <font color='#5555FF'>+</font><font color='#5555FF'>+</font>num_relus; <b>}</b><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>num_relus <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>10</font><font face='Lucida Console'>)</font>;
        num_relus <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        <font color='#BB00BB'>visit_computational_layers</font><font face='Lucida Console'>(</font>const_pnet, [<font color='#5555FF'>&amp;</font>num_relus]<font face='Lucida Console'>(</font><font color='#0000FF'>const</font> relu_<font color='#5555FF'>&amp;</font><font face='Lucida Console'>)</font> <b>{</b> <font color='#5555FF'>+</font><font color='#5555FF'>+</font>num_relus; <b>}</b><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>num_relus <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>10</font><font face='Lucida Console'>)</font>;
        num_relus <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        <font color='#BB00BB'>visit_computational_layers</font><font face='Lucida Console'>(</font>const_pnet, [<font color='#5555FF'>&amp;</font>num_relus]<font face='Lucida Console'>(</font>relu_<font color='#5555FF'>&amp;</font><font face='Lucida Console'>)</font> <b>{</b> <font color='#5555FF'>+</font><font color='#5555FF'>+</font>num_relus; <b>}</b><font face='Lucida Console'>)</font>;
        <font color='#009900'>// Visiting doesn't happen in this case because a const network can't bind the non-const
</font>        <font color='#009900'>// relu_ reference used above. 
</font>        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>num_relus <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;

        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>layer<font color='#5555FF'>&lt;</font>leaky_relu<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>pnet<font face='Lucida Console'>)</font>.<font color='#BB00BB'>layer_details</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_alpha</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0.01f</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>visit_computational_layers</font><font face='Lucida Console'>(</font>pnet, []<font face='Lucida Console'>(</font>leaky_relu_<font color='#5555FF'>&amp;</font> l<font face='Lucida Console'>)</font> <b>{</b> l <font color='#5555FF'>=</font> <font color='#BB00BB'>leaky_relu_</font><font face='Lucida Console'>(</font><font color='#979000'>0.001f</font><font face='Lucida Console'>)</font>; <b>}</b><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>layer<font color='#5555FF'>&lt;</font>leaky_relu<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>pnet<font face='Lucida Console'>)</font>.<font color='#BB00BB'>layer_details</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_alpha</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0.001f</font><font face='Lucida Console'>)</font>;

        <font color='#009900'>// make sure count_parameters() works since it depends on visiting too.  Initially the
</font>        <font color='#009900'>// network has 0 parameters.  But once we run something through it it will allocate its
</font>        <font color='#009900'>// parameters.
</font>        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font><font color='#BB00BB'>count_parameters</font><font face='Lucida Console'>(</font>pnet<font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0</font>, "<font color='#CC0000'>count_parameters(pnet): </font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>count_parameters</font><font face='Lucida Console'>(</font>pnet<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <font color='#0000FF'>const</font> matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>char</u></font><font color='#5555FF'>&gt;</font> input <font color='#5555FF'>=</font> zeros_matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>char</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font><font color='#979000'>40</font>,<font color='#979000'>40</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>pnet</font><font face='Lucida Console'>(</font>input<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font><font color='#BB00BB'>count_parameters</font><font face='Lucida Console'>(</font>pnet<font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>17606</font>, "<font color='#CC0000'>count_parameters(pnet): </font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>count_parameters</font><font face='Lucida Console'>(</font>pnet<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

    <b>}</b>

    <font color='#0000FF'><u>float</u></font> <b><a name='tensor_read_cpu'></a>tensor_read_cpu</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> t, <font color='#0000FF'><u>long</u></font> i, <font color='#0000FF'><u>long</u></font> k, <font color='#0000FF'><u>long</u></font> r, <font color='#0000FF'><u>long</u></font> c<font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> p <font color='#5555FF'>=</font> t.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> t.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>*</font> t.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>*</font> t.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>*</font> i <font color='#5555FF'>+</font>
                        t.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>*</font> t.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>*</font> k <font color='#5555FF'>+</font> t.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>*</font> r <font color='#5555FF'>+</font> c;
        <font color='#0000FF'>return</font> <font color='#5555FF'>*</font>p;
    <b>}</b>
    <font color='#0000FF'><u>void</u></font> <b><a name='test_copy_tensor_cpu'></a>test_copy_tensor_cpu</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#0000FF'>using</font> <font color='#0000FF'>namespace</font> dlib::tt;
        <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        resizable_tensor <font color='#BB00BB'>dest</font><font face='Lucida Console'>(</font><font color='#979000'>10</font>, <font color='#979000'>9</font>, <font color='#979000'>7</font>, <font color='#979000'>15</font><font face='Lucida Console'>)</font>;
        resizable_tensor <font color='#BB00BB'>src1</font><font face='Lucida Console'>(</font><font color='#979000'>10</font>, <font color='#979000'>3</font>, <font color='#979000'>7</font>, <font color='#979000'>15</font><font face='Lucida Console'>)</font>;
        resizable_tensor <font color='#BB00BB'>src2</font><font face='Lucida Console'>(</font><font color='#979000'>10</font>, <font color='#979000'>3</font>, <font color='#979000'>7</font>, <font color='#979000'>15</font><font face='Lucida Console'>)</font>;
        resizable_tensor <font color='#BB00BB'>src3</font><font face='Lucida Console'>(</font><font color='#979000'>10</font>, <font color='#979000'>9</font>, <font color='#979000'>7</font>, <font color='#979000'>15</font><font face='Lucida Console'>)</font>;
        tt::tensor_rand rnd;
        rnd.<font color='#BB00BB'>fill_gaussian</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font>;
        rnd.<font color='#BB00BB'>fill_gaussian</font><font face='Lucida Console'>(</font>src1<font face='Lucida Console'>)</font>;
        rnd.<font color='#BB00BB'>fill_gaussian</font><font face='Lucida Console'>(</font>src2<font face='Lucida Console'>)</font>;
        rnd.<font color='#BB00BB'>fill_gaussian</font><font face='Lucida Console'>(</font>src3<font face='Lucida Console'>)</font>;

        cpu::<font color='#BB00BB'>copy_tensor</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, dest, <font color='#979000'>0</font>, src1, <font color='#979000'>0</font>,  src1.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>; <font color='#009900'>//full copy src1-&gt;dest
</font>        cpu::<font color='#BB00BB'>copy_tensor</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, dest, src1.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, src2, <font color='#979000'>0</font>,  src2.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>; <font color='#009900'>//full copy src2-&gt;dest with offset of src1
</font>        cpu::<font color='#BB00BB'>copy_tensor</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, dest, src1.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> src2.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, src3, <font color='#979000'>3</font>,  <font color='#979000'>3</font><font face='Lucida Console'>)</font>; <font color='#009900'>//partial copy src3 into the rest place of dest
</font>

        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> dest.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'>&lt;</font> dest.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> r <font color='#5555FF'>=</font> <font color='#979000'>0</font>; r <font color='#5555FF'>&lt;</font> dest.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>r<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> c <font color='#5555FF'>=</font> <font color='#979000'>0</font>; c <font color='#5555FF'>&lt;</font> dest.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>c<font face='Lucida Console'>)</font>
                    <b>{</b>
                        <font color='#0000FF'><u>float</u></font> dest_value <font color='#5555FF'>=</font> <font color='#BB00BB'>tensor_read_cpu</font><font face='Lucida Console'>(</font>dest, i, k, r, c<font face='Lucida Console'>)</font>;
                        <font color='#009900'>// first part is from src1
</font>                        <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>k <font color='#5555FF'>&lt;</font> src1.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                        <b>{</b>
                            <font color='#0000FF'><u>float</u></font> src_value <font color='#5555FF'>=</font> <font color='#BB00BB'>tensor_read_cpu</font><font face='Lucida Console'>(</font>src1, i, k, r, c<font face='Lucida Console'>)</font>;
                            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>src_value <font color='#5555FF'>=</font><font color='#5555FF'>=</font> dest_value<font face='Lucida Console'>)</font>;
                        <b>}</b>
                        <font color='#009900'>// second part is from src2
</font>                        <font color='#0000FF'>else</font> <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>k <font color='#5555FF'>&lt;</font> src1.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> src2.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                        <b>{</b>
                            <font color='#0000FF'><u>float</u></font> src_value <font color='#5555FF'>=</font> <font color='#BB00BB'>tensor_read_cpu</font><font face='Lucida Console'>(</font>src2, i, k <font color='#5555FF'>-</font> src1.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, r, c<font face='Lucida Console'>)</font>;
                            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>src_value <font color='#5555FF'>=</font><font color='#5555FF'>=</font> dest_value<font face='Lucida Console'>)</font>;
                        <b>}</b>
                        <font color='#009900'>// third part is from src3
</font>                        <font color='#0000FF'>else</font>
                        <b>{</b>
                            <font color='#0000FF'><u>float</u></font> src_value <font color='#5555FF'>=</font> <font color='#BB00BB'>tensor_read_cpu</font><font face='Lucida Console'>(</font>src3, i, k <font color='#5555FF'>-</font> src1.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> src2.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> <font color='#979000'>3</font>, r, c<font face='Lucida Console'>)</font>;
                            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>src_value <font color='#5555FF'>=</font><font color='#5555FF'>=</font> dest_value<font face='Lucida Console'>)</font>;
                        <b>}</b>
                    <b>}</b>
                <b>}</b>
            <b>}</b>
        <b>}</b>
    <b>}</b>
    <font color='#0000FF'><u>void</u></font> <b><a name='test_copy_tensor_add_to_cpu'></a>test_copy_tensor_add_to_cpu</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#0000FF'>using</font> <font color='#0000FF'>namespace</font> dlib::tt;
        <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        resizable_tensor <font color='#BB00BB'>dest</font><font face='Lucida Console'>(</font><font color='#979000'>10</font>, <font color='#979000'>9</font>, <font color='#979000'>7</font>, <font color='#979000'>15</font><font face='Lucida Console'>)</font>;
        resizable_tensor <font color='#BB00BB'>src1</font><font face='Lucida Console'>(</font><font color='#979000'>10</font>, <font color='#979000'>3</font>, <font color='#979000'>7</font>, <font color='#979000'>15</font><font face='Lucida Console'>)</font>;
        resizable_tensor <font color='#BB00BB'>src2</font><font face='Lucida Console'>(</font><font color='#979000'>10</font>, <font color='#979000'>3</font>, <font color='#979000'>7</font>, <font color='#979000'>15</font><font face='Lucida Console'>)</font>;
        resizable_tensor <font color='#BB00BB'>src3</font><font face='Lucida Console'>(</font><font color='#979000'>10</font>, <font color='#979000'>9</font>, <font color='#979000'>7</font>, <font color='#979000'>15</font><font face='Lucida Console'>)</font>;
        tt::tensor_rand rnd;
        rnd.<font color='#BB00BB'>fill_gaussian</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font>;
        rnd.<font color='#BB00BB'>fill_gaussian</font><font face='Lucida Console'>(</font>src1<font face='Lucida Console'>)</font>;
        rnd.<font color='#BB00BB'>fill_gaussian</font><font face='Lucida Console'>(</font>src2<font face='Lucida Console'>)</font>;
        rnd.<font color='#BB00BB'>fill_gaussian</font><font face='Lucida Console'>(</font>src3<font face='Lucida Console'>)</font>;

        <font color='#0000FF'>const</font> resizable_tensor old_dest <font color='#5555FF'>=</font> dest;

        cpu::<font color='#BB00BB'>copy_tensor</font><font face='Lucida Console'>(</font><font color='#979000'>true</font>, dest, <font color='#979000'>0</font>, src1, <font color='#979000'>0</font>,  src1.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>; <font color='#009900'>//full copy src1-&gt;dest
</font>        cpu::<font color='#BB00BB'>copy_tensor</font><font face='Lucida Console'>(</font><font color='#979000'>true</font>, dest, src1.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, src2, <font color='#979000'>0</font>,  src2.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>; <font color='#009900'>//full copy src2-&gt;dest with offset of src1
</font>        cpu::<font color='#BB00BB'>copy_tensor</font><font face='Lucida Console'>(</font><font color='#979000'>true</font>, dest, src1.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> src2.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, src3, <font color='#979000'>3</font>,  <font color='#979000'>3</font><font face='Lucida Console'>)</font>; <font color='#009900'>//partial copy src3 into the rest place of dest
</font>

        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> dest.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'>&lt;</font> dest.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> r <font color='#5555FF'>=</font> <font color='#979000'>0</font>; r <font color='#5555FF'>&lt;</font> dest.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>r<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> c <font color='#5555FF'>=</font> <font color='#979000'>0</font>; c <font color='#5555FF'>&lt;</font> dest.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>c<font face='Lucida Console'>)</font>
                    <b>{</b>
                        <font color='#0000FF'><u>float</u></font> old_dest_value <font color='#5555FF'>=</font> <font color='#BB00BB'>tensor_read_cpu</font><font face='Lucida Console'>(</font>old_dest, i, k, r, c<font face='Lucida Console'>)</font>;
                        <font color='#0000FF'><u>float</u></font> dest_value <font color='#5555FF'>=</font> <font color='#BB00BB'>tensor_read_cpu</font><font face='Lucida Console'>(</font>dest, i, k, r, c<font face='Lucida Console'>)</font>;
                        <font color='#009900'>// first part is from src1
</font>                        <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>k <font color='#5555FF'>&lt;</font> src1.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                        <b>{</b>
                            <font color='#0000FF'><u>float</u></font> src_value <font color='#5555FF'>=</font> <font color='#BB00BB'>tensor_read_cpu</font><font face='Lucida Console'>(</font>src1, i, k, r, c<font face='Lucida Console'>)</font><font color='#5555FF'>+</font>old_dest_value;
                            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>src_value <font color='#5555FF'>-</font> dest_value<font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font><font face='Lucida Console'>)</font>;
                        <b>}</b>
                        <font color='#009900'>// second part is from src2
</font>                        <font color='#0000FF'>else</font> <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>k <font color='#5555FF'>&lt;</font> src1.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> src2.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                        <b>{</b>
                            <font color='#0000FF'><u>float</u></font> src_value <font color='#5555FF'>=</font> <font color='#BB00BB'>tensor_read_cpu</font><font face='Lucida Console'>(</font>src2, i, k <font color='#5555FF'>-</font> src1.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, r, c<font face='Lucida Console'>)</font><font color='#5555FF'>+</font>old_dest_value;
                            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>src_value <font color='#5555FF'>-</font> dest_value<font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font><font face='Lucida Console'>)</font>;
                        <b>}</b>
                        <font color='#009900'>// third part is from src3
</font>                        <font color='#0000FF'>else</font>
                        <b>{</b>
                            <font color='#0000FF'><u>float</u></font> src_value <font color='#5555FF'>=</font> <font color='#BB00BB'>tensor_read_cpu</font><font face='Lucida Console'>(</font>src3, i, k <font color='#5555FF'>-</font> src1.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> src2.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> <font color='#979000'>3</font>, r, c<font face='Lucida Console'>)</font><font color='#5555FF'>+</font>old_dest_value;
                            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>src_value <font color='#5555FF'>-</font> dest_value<font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font><font face='Lucida Console'>)</font>;
                        <b>}</b>
                    <b>}</b>
                <b>}</b>
            <b>}</b>
        <b>}</b>
    <b>}</b>
<font color='#0000FF'>#ifdef</font> DLIB_USE_CUDA
    <font color='#0000FF'><u>void</u></font> <b><a name='test_copy_tensor_gpu'></a>test_copy_tensor_gpu</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#0000FF'>using</font> <font color='#0000FF'>namespace</font> dlib::tt;
        <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        resizable_tensor <font color='#BB00BB'>dest</font><font face='Lucida Console'>(</font><font color='#979000'>10</font>, <font color='#979000'>9</font>, <font color='#979000'>7</font>, <font color='#979000'>15</font><font face='Lucida Console'>)</font>;
        resizable_tensor <font color='#BB00BB'>src1</font><font face='Lucida Console'>(</font><font color='#979000'>10</font>, <font color='#979000'>3</font>, <font color='#979000'>7</font>, <font color='#979000'>15</font><font face='Lucida Console'>)</font>;
        resizable_tensor <font color='#BB00BB'>src2</font><font face='Lucida Console'>(</font><font color='#979000'>10</font>, <font color='#979000'>3</font>, <font color='#979000'>7</font>, <font color='#979000'>15</font><font face='Lucida Console'>)</font>;
        resizable_tensor <font color='#BB00BB'>src3</font><font face='Lucida Console'>(</font><font color='#979000'>10</font>, <font color='#979000'>9</font>, <font color='#979000'>7</font>, <font color='#979000'>15</font><font face='Lucida Console'>)</font>;
        tt::tensor_rand rnd;
        rnd.<font color='#BB00BB'>fill_gaussian</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font>;
        rnd.<font color='#BB00BB'>fill_gaussian</font><font face='Lucida Console'>(</font>src1<font face='Lucida Console'>)</font>;
        rnd.<font color='#BB00BB'>fill_gaussian</font><font face='Lucida Console'>(</font>src2<font face='Lucida Console'>)</font>;
        rnd.<font color='#BB00BB'>fill_gaussian</font><font face='Lucida Console'>(</font>src3<font face='Lucida Console'>)</font>;
        cuda::<font color='#BB00BB'>copy_tensor</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, dest, <font color='#979000'>0</font>, src1, <font color='#979000'>0</font>,  src1.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>; <font color='#009900'>//full copy src1-&gt;dest
</font>        cuda::<font color='#BB00BB'>copy_tensor</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, dest, src1.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, src2, <font color='#979000'>0</font>,  src2.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>; <font color='#009900'>//full copy src2-&gt;dest with offset of src1
</font>        cuda::<font color='#BB00BB'>copy_tensor</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, dest, src1.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> src2.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, src3, <font color='#979000'>3</font>,  <font color='#979000'>3</font><font face='Lucida Console'>)</font>; <font color='#009900'>//partial copy src3 into the rest place of dest
</font>

        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> dest.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'>&lt;</font> dest.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> r <font color='#5555FF'>=</font> <font color='#979000'>0</font>; r <font color='#5555FF'>&lt;</font> dest.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>r<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> c <font color='#5555FF'>=</font> <font color='#979000'>0</font>; c <font color='#5555FF'>&lt;</font> dest.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>c<font face='Lucida Console'>)</font>
                    <b>{</b>
                        <font color='#0000FF'><u>float</u></font> dest_value <font color='#5555FF'>=</font> <font color='#BB00BB'>tensor_read_cpu</font><font face='Lucida Console'>(</font>dest, i, k, r, c<font face='Lucida Console'>)</font>;
                        <font color='#009900'>// first part is from src1
</font>                        <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>k <font color='#5555FF'>&lt;</font> src1.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                        <b>{</b>
                            <font color='#0000FF'><u>float</u></font> src_value <font color='#5555FF'>=</font> <font color='#BB00BB'>tensor_read_cpu</font><font face='Lucida Console'>(</font>src1, i, k, r, c<font face='Lucida Console'>)</font>;
                            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>src_value <font color='#5555FF'>=</font><font color='#5555FF'>=</font> dest_value<font face='Lucida Console'>)</font>;
                        <b>}</b>
                            <font color='#009900'>// second part is from src2
</font>                        <font color='#0000FF'>else</font> <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>k <font color='#5555FF'>&lt;</font> src1.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> src2.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                        <b>{</b>
                            <font color='#0000FF'><u>float</u></font> src_value <font color='#5555FF'>=</font> <font color='#BB00BB'>tensor_read_cpu</font><font face='Lucida Console'>(</font>src2, i, k <font color='#5555FF'>-</font> src1.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, r, c<font face='Lucida Console'>)</font>;
                            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>src_value <font color='#5555FF'>=</font><font color='#5555FF'>=</font> dest_value<font face='Lucida Console'>)</font>;
                        <b>}</b>
                            <font color='#009900'>// third part is from src3
</font>                        <font color='#0000FF'>else</font>
                        <b>{</b>
                            <font color='#0000FF'><u>float</u></font> src_value <font color='#5555FF'>=</font> <font color='#BB00BB'>tensor_read_cpu</font><font face='Lucida Console'>(</font>src3, i, k <font color='#5555FF'>-</font> src1.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> src2.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> <font color='#979000'>3</font>, r, c<font face='Lucida Console'>)</font>;
                            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>src_value <font color='#5555FF'>=</font><font color='#5555FF'>=</font> dest_value<font face='Lucida Console'>)</font>;
                        <b>}</b>
                    <b>}</b>
                <b>}</b>
            <b>}</b>
        <b>}</b>
    <b>}</b>
    <font color='#0000FF'><u>void</u></font> <b><a name='test_copy_tensor_add_to_gpu'></a>test_copy_tensor_add_to_gpu</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#0000FF'>using</font> <font color='#0000FF'>namespace</font> dlib::tt;
        <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        resizable_tensor <font color='#BB00BB'>dest</font><font face='Lucida Console'>(</font><font color='#979000'>10</font>, <font color='#979000'>9</font>, <font color='#979000'>7</font>, <font color='#979000'>15</font><font face='Lucida Console'>)</font>;
        resizable_tensor <font color='#BB00BB'>src1</font><font face='Lucida Console'>(</font><font color='#979000'>10</font>, <font color='#979000'>3</font>, <font color='#979000'>7</font>, <font color='#979000'>15</font><font face='Lucida Console'>)</font>;
        resizable_tensor <font color='#BB00BB'>src2</font><font face='Lucida Console'>(</font><font color='#979000'>10</font>, <font color='#979000'>3</font>, <font color='#979000'>7</font>, <font color='#979000'>15</font><font face='Lucida Console'>)</font>;
        resizable_tensor <font color='#BB00BB'>src3</font><font face='Lucida Console'>(</font><font color='#979000'>10</font>, <font color='#979000'>9</font>, <font color='#979000'>7</font>, <font color='#979000'>15</font><font face='Lucida Console'>)</font>;
        tt::tensor_rand rnd;
        rnd.<font color='#BB00BB'>fill_gaussian</font><font face='Lucida Console'>(</font>dest<font face='Lucida Console'>)</font>;
        rnd.<font color='#BB00BB'>fill_gaussian</font><font face='Lucida Console'>(</font>src1<font face='Lucida Console'>)</font>;
        rnd.<font color='#BB00BB'>fill_gaussian</font><font face='Lucida Console'>(</font>src2<font face='Lucida Console'>)</font>;
        rnd.<font color='#BB00BB'>fill_gaussian</font><font face='Lucida Console'>(</font>src3<font face='Lucida Console'>)</font>;

        <font color='#0000FF'>const</font> resizable_tensor old_dest <font color='#5555FF'>=</font> dest;

        cuda::<font color='#BB00BB'>copy_tensor</font><font face='Lucida Console'>(</font><font color='#979000'>true</font>, dest, <font color='#979000'>0</font>, src1, <font color='#979000'>0</font>,  src1.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>; <font color='#009900'>//full copy src1-&gt;dest
</font>        cuda::<font color='#BB00BB'>copy_tensor</font><font face='Lucida Console'>(</font><font color='#979000'>true</font>, dest, src1.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, src2, <font color='#979000'>0</font>,  src2.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>; <font color='#009900'>//full copy src2-&gt;dest with offset of src1
</font>        cuda::<font color='#BB00BB'>copy_tensor</font><font face='Lucida Console'>(</font><font color='#979000'>true</font>, dest, src1.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> src2.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, src3, <font color='#979000'>3</font>,  <font color='#979000'>3</font><font face='Lucida Console'>)</font>; <font color='#009900'>//partial copy src3 into the rest place of dest
</font>

        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> dest.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'>&lt;</font> dest.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> r <font color='#5555FF'>=</font> <font color='#979000'>0</font>; r <font color='#5555FF'>&lt;</font> dest.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>r<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> c <font color='#5555FF'>=</font> <font color='#979000'>0</font>; c <font color='#5555FF'>&lt;</font> dest.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>c<font face='Lucida Console'>)</font>
                    <b>{</b>
                        <font color='#0000FF'><u>float</u></font> old_dest_value <font color='#5555FF'>=</font> <font color='#BB00BB'>tensor_read_cpu</font><font face='Lucida Console'>(</font>old_dest, i, k, r, c<font face='Lucida Console'>)</font>;
                        <font color='#0000FF'><u>float</u></font> dest_value <font color='#5555FF'>=</font> <font color='#BB00BB'>tensor_read_cpu</font><font face='Lucida Console'>(</font>dest, i, k, r, c<font face='Lucida Console'>)</font>;
                        <font color='#009900'>// first part is from src1
</font>                        <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>k <font color='#5555FF'>&lt;</font> src1.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                        <b>{</b>
                            <font color='#0000FF'><u>float</u></font> src_value <font color='#5555FF'>=</font> <font color='#BB00BB'>tensor_read_cpu</font><font face='Lucida Console'>(</font>src1, i, k, r, c<font face='Lucida Console'>)</font><font color='#5555FF'>+</font>old_dest_value;
                            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>src_value <font color='#5555FF'>-</font> dest_value<font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font>, std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>src_value <font color='#5555FF'>-</font> dest_value<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                        <b>}</b>
                            <font color='#009900'>// second part is from src2
</font>                        <font color='#0000FF'>else</font> <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>k <font color='#5555FF'>&lt;</font> src1.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> src2.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                        <b>{</b>
                            <font color='#0000FF'><u>float</u></font> src_value <font color='#5555FF'>=</font> <font color='#BB00BB'>tensor_read_cpu</font><font face='Lucida Console'>(</font>src2, i, k <font color='#5555FF'>-</font> src1.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, r, c<font face='Lucida Console'>)</font><font color='#5555FF'>+</font>old_dest_value;
                            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>src_value <font color='#5555FF'>-</font> dest_value<font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font><font face='Lucida Console'>)</font>;
                        <b>}</b>
                            <font color='#009900'>// third part is from src3
</font>                        <font color='#0000FF'>else</font>
                        <b>{</b>
                            <font color='#0000FF'><u>float</u></font> src_value <font color='#5555FF'>=</font> <font color='#BB00BB'>tensor_read_cpu</font><font face='Lucida Console'>(</font>src3, i, k <font color='#5555FF'>-</font> src1.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> src2.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> <font color='#979000'>3</font>, r, c<font face='Lucida Console'>)</font><font color='#5555FF'>+</font>old_dest_value;
                            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>src_value <font color='#5555FF'>-</font> dest_value<font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font><font face='Lucida Console'>)</font>;
                        <b>}</b>
                    <b>}</b>
                <b>}</b>
            <b>}</b>
        <b>}</b>
    <b>}</b>
<font color='#0000FF'>#endif</font><font color='#009900'>//DLIB_USE_CUDA
</font>
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> concat_block1 <font color='#5555FF'>=</font> con<font color='#5555FF'>&lt;</font><font color='#979000'>5</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> concat_block2 <font color='#5555FF'>=</font> con<font color='#5555FF'>&lt;</font><font color='#979000'>8</font>,<font color='#979000'>3</font>,<font color='#979000'>3</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> concat_block3 <font color='#5555FF'>=</font> max_pool<font color='#5555FF'>&lt;</font><font color='#979000'>3</font>,<font color='#979000'>3</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> concat_incept <font color='#5555FF'>=</font> inception3<font color='#5555FF'>&lt;</font>concat_block1,concat_block2,concat_block3,SUBNET<font color='#5555FF'>&gt;</font>;

    <font color='#0000FF'><u>void</u></font> <b><a name='test_concat'></a>test_concat</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#0000FF'>using</font> <font color='#0000FF'>namespace</font> dlib::tt;
        <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

        <font color='#0000FF'>using</font> net_type <font color='#5555FF'>=</font> concat_incept<font color='#5555FF'>&lt;</font>input<font color='#5555FF'>&lt;</font>matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>;

        resizable_tensor <font color='#BB00BB'>data</font><font face='Lucida Console'>(</font><font color='#979000'>10</font>, <font color='#979000'>1</font>, <font color='#979000'>111</font>, <font color='#979000'>222</font><font face='Lucida Console'>)</font>;
        tt::tensor_rand rnd;
        rnd.<font color='#BB00BB'>fill_gaussian</font><font face='Lucida Console'>(</font>data<font face='Lucida Console'>)</font>;

        net_type net;


        <font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font> out <font color='#5555FF'>=</font> net.<font color='#BB00BB'>forward</font><font face='Lucida Console'>(</font>data<font face='Lucida Console'>)</font>;

        <font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font> b1o <font color='#5555FF'>=</font> layer<font color='#5555FF'>&lt;</font>itag1<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font> b2o <font color='#5555FF'>=</font> layer<font color='#5555FF'>&lt;</font>itag2<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font> b3o <font color='#5555FF'>=</font> layer<font color='#5555FF'>&lt;</font>itag3<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

        resizable_tensor <font color='#BB00BB'>dest</font><font face='Lucida Console'>(</font><font color='#979000'>10</font>, <font color='#979000'>14</font>, <font color='#979000'>111</font>, <font color='#979000'>222</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>copy_tensor</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, dest, <font color='#979000'>0</font>, b1o, <font color='#979000'>0</font>,  b1o.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>copy_tensor</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, dest, b1o.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, b2o, <font color='#979000'>0</font>,  b2o.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>copy_tensor</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, dest, b1o.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> b2o.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, b3o, <font color='#979000'>0</font>,  b3o.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>dest.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> out.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <font color='#0000FF'><u>int</u></font> error <font color='#5555FF'>=</font> <font color='#BB00BB'>memcmp</font><font face='Lucida Console'>(</font>dest.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, out.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, dest.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>error <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;

        resizable_tensor <font color='#BB00BB'>gr</font><font face='Lucida Console'>(</font><font color='#979000'>10</font>, <font color='#979000'>14</font>, <font color='#979000'>111</font>, <font color='#979000'>222</font><font face='Lucida Console'>)</font>;
        rnd.<font color='#BB00BB'>fill_gaussian</font><font face='Lucida Console'>(</font>gr<font face='Lucida Console'>)</font>;

        resizable_tensor params;
        net.<font color='#BB00BB'>layer_details</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>backward</font><font face='Lucida Console'>(</font>gr, net, params<font face='Lucida Console'>)</font>;

        <font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font> b1g <font color='#5555FF'>=</font> layer<font color='#5555FF'>&lt;</font>itag1<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font>.<font color='#BB00BB'>subnet</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font> b2g <font color='#5555FF'>=</font> layer<font color='#5555FF'>&lt;</font>itag2<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font>.<font color='#BB00BB'>subnet</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font> b3g <font color='#5555FF'>=</font> layer<font color='#5555FF'>&lt;</font>itag3<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font>.<font color='#BB00BB'>subnet</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

        resizable_tensor <font color='#BB00BB'>g1</font><font face='Lucida Console'>(</font><font color='#979000'>10</font>, <font color='#979000'>5</font>, <font color='#979000'>111</font>, <font color='#979000'>222</font><font face='Lucida Console'>)</font>;
        resizable_tensor <font color='#BB00BB'>g2</font><font face='Lucida Console'>(</font><font color='#979000'>10</font>, <font color='#979000'>8</font>, <font color='#979000'>111</font>, <font color='#979000'>222</font><font face='Lucida Console'>)</font>;
        resizable_tensor <font color='#BB00BB'>g3</font><font face='Lucida Console'>(</font><font color='#979000'>10</font>, <font color='#979000'>1</font>, <font color='#979000'>111</font>, <font color='#979000'>222</font><font face='Lucida Console'>)</font>;

        <font color='#BB00BB'>copy_tensor</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, g1, <font color='#979000'>0</font>, gr, <font color='#979000'>0</font>,  g1.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>copy_tensor</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, g2, <font color='#979000'>0</font>, gr, g1.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, g2.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>copy_tensor</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, g3, <font color='#979000'>0</font>, gr, g1.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> g2.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, g3.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>g1.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> b1g.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        error <font color='#5555FF'>=</font> <font color='#BB00BB'>memcmp</font><font face='Lucida Console'>(</font>g1.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, b1g.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, b1g.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>error <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>g2.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> b2g.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        error <font color='#5555FF'>=</font> <font color='#BB00BB'>memcmp</font><font face='Lucida Console'>(</font>g2.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, b2g.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, b2g.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>error <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>g3.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> b3g.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        error <font color='#5555FF'>=</font> <font color='#BB00BB'>memcmp</font><font face='Lucida Console'>(</font>g3.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, b3g.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, b3g.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>error <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
    <b>}</b>

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'><u>void</u></font> <b><a name='test_simple_linear_regression'></a>test_simple_linear_regression</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>int</u></font> num_samples <font color='#5555FF'>=</font> <font color='#979000'>1000</font>;
        ::std::vector<font color='#5555FF'>&lt;</font>matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font> <font color='#BB00BB'>x</font><font face='Lucida Console'>(</font>num_samples<font face='Lucida Console'>)</font>;
        ::std::vector<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font> <font color='#BB00BB'>y</font><font face='Lucida Console'>(</font>num_samples<font face='Lucida Console'>)</font>;
        ::std::default_random_engine <font color='#BB00BB'>generator</font><font face='Lucida Console'>(</font><font color='#979000'>16</font><font face='Lucida Console'>)</font>;
        ::std::normal_distribution<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font> <font color='#BB00BB'>distribution</font><font face='Lucida Console'>(</font><font color='#979000'>0</font>,<font color='#979000'>0.1</font><font face='Lucida Console'>)</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> true_intercept <font color='#5555FF'>=</font> <font color='#979000'>50.0</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> true_slope <font color='#5555FF'>=</font> <font color='#979000'>10.0</font>;
        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font> <font color='#0000FF'><u>int</u></font> ii <font color='#5555FF'>=</font> <font color='#979000'>0</font>; ii <font color='#5555FF'>&lt;</font> num_samples; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>ii <font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> val <font color='#5555FF'>=</font> <font color='#0000FF'>static_cast</font><font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>ii<font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font color='#979000'>10</font>;
            matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font><font color='#5555FF'>&gt;</font> <font color='#BB00BB'>tmp</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,<font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            tmp <font color='#5555FF'>=</font> val;
            x[ii] <font color='#5555FF'>=</font> tmp;
            y[ii] <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font>true_intercept <font color='#5555FF'>+</font> true_slope<font color='#5555FF'>*</font><font color='#0000FF'>static_cast</font><font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>val<font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> <font color='#BB00BB'>distribution</font><font face='Lucida Console'>(</font>generator<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>using</font> net_type <font color='#5555FF'>=</font> loss_mean_squared<font color='#5555FF'>&lt;</font>fc<font color='#5555FF'>&lt;</font><font color='#979000'>1</font>, input<font color='#5555FF'>&lt;</font>matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>;
        net_type net;
        layer<font color='#5555FF'>&lt;</font><font color='#979000'>1</font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font>.<font color='#BB00BB'>layer_details</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>set_bias_learning_rate_multiplier</font><font face='Lucida Console'>(</font><font color='#979000'>300</font><font face='Lucida Console'>)</font>;
        sgd <font color='#BB00BB'>defsolver</font><font face='Lucida Console'>(</font><font color='#979000'>0</font>,<font color='#979000'>0.9</font><font face='Lucida Console'>)</font>;
        dnn_trainer<font color='#5555FF'>&lt;</font>net_type<font color='#5555FF'>&gt;</font> <font color='#BB00BB'>trainer</font><font face='Lucida Console'>(</font>net, defsolver<font face='Lucida Console'>)</font>;
        trainer.<font color='#BB00BB'>set_learning_rate</font><font face='Lucida Console'>(</font><font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;
        trainer.<font color='#BB00BB'>set_min_learning_rate</font><font face='Lucida Console'>(</font><font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font><font face='Lucida Console'>)</font>;
        trainer.<font color='#BB00BB'>set_mini_batch_size</font><font face='Lucida Console'>(</font><font color='#979000'>50</font><font face='Lucida Console'>)</font>;
        trainer.<font color='#BB00BB'>set_max_num_epochs</font><font face='Lucida Console'>(</font><font color='#979000'>170</font><font face='Lucida Console'>)</font>;
        trainer.<font color='#BB00BB'>train</font><font face='Lucida Console'>(</font>x, y<font face='Lucida Console'>)</font>;

        <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> slope <font color='#5555FF'>=</font> layer<font color='#5555FF'>&lt;</font><font color='#979000'>1</font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font>.<font color='#BB00BB'>layer_details</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_weights</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[<font color='#979000'>0</font>];
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> slope_error <font color='#5555FF'>=</font> <font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>true_slope <font color='#5555FF'>-</font> slope<font face='Lucida Console'>)</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> intercept <font color='#5555FF'>=</font> layer<font color='#5555FF'>&lt;</font><font color='#979000'>1</font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font>.<font color='#BB00BB'>layer_details</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_biases</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[<font color='#979000'>0</font>];
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> intercept_error <font color='#5555FF'>=</font> <font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>true_intercept <font color='#5555FF'>-</font> intercept<font face='Lucida Console'>)</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> eps_slope <font color='#5555FF'>=</font> <font color='#979000'>0.05</font>, eps_intercept <font color='#5555FF'>=</font> <font color='#979000'>0.1</font>;

        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>slope_error <font color='#5555FF'>&lt;</font><font color='#5555FF'>=</font> eps_slope,
                      "<font color='#CC0000'>Expected slope = </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> true_slope <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> Estimated slope = </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> slope <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> Error limit = </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> eps_slope<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>intercept_error <font color='#5555FF'>&lt;</font><font color='#5555FF'>=</font> eps_intercept,
                      "<font color='#CC0000'>Expected intercept = </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> true_intercept <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> Estimated intercept = </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> intercept <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> Error limit = </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> eps_intercept<font face='Lucida Console'>)</font>;

    <b>}</b>

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'><u>void</u></font> <b><a name='test_simple_linear_regression_eil'></a>test_simple_linear_regression_eil</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>int</u></font> num_samples <font color='#5555FF'>=</font> <font color='#979000'>1000</font>;
        ::std::vector<font color='#5555FF'>&lt;</font>matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font> <font color='#BB00BB'>x</font><font face='Lucida Console'>(</font>num_samples<font face='Lucida Console'>)</font>;
        ::std::vector<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font> <font color='#BB00BB'>y</font><font face='Lucida Console'>(</font>num_samples<font face='Lucida Console'>)</font>;
        ::std::default_random_engine <font color='#BB00BB'>generator</font><font face='Lucida Console'>(</font><font color='#979000'>16</font><font face='Lucida Console'>)</font>;
        ::std::normal_distribution<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font> <font color='#BB00BB'>distribution</font><font face='Lucida Console'>(</font><font color='#979000'>0</font>,<font color='#979000'>0.0001</font><font face='Lucida Console'>)</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> true_intercept <font color='#5555FF'>=</font> <font color='#979000'>50.0</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> true_slope <font color='#5555FF'>=</font> <font color='#979000'>10.0</font>;
        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font> <font color='#0000FF'><u>int</u></font> ii <font color='#5555FF'>=</font> <font color='#979000'>0</font>; ii <font color='#5555FF'>&lt;</font> num_samples; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>ii <font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> val <font color='#5555FF'>=</font> <font color='#0000FF'>static_cast</font><font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>ii<font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font color='#979000'>10</font>;
            matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font><font color='#5555FF'>&gt;</font> <font color='#BB00BB'>tmp</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,<font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            tmp <font color='#5555FF'>=</font> val;
            x[ii] <font color='#5555FF'>=</font> tmp;
            y[ii] <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font>true_intercept <font color='#5555FF'>+</font> true_slope<font color='#5555FF'>*</font><font color='#0000FF'>static_cast</font><font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>val<font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> <font color='#BB00BB'>distribution</font><font face='Lucida Console'>(</font>generator<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>using</font> net_type <font color='#5555FF'>=</font> loss_epsilon_insensitive<font color='#5555FF'>&lt;</font>fc<font color='#5555FF'>&lt;</font><font color='#979000'>1</font>, input<font color='#5555FF'>&lt;</font>matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>;
        net_type <font color='#BB00BB'>net</font><font face='Lucida Console'>(</font><font color='#979000'>0.01</font><font face='Lucida Console'>)</font>;
        layer<font color='#5555FF'>&lt;</font><font color='#979000'>1</font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font>.<font color='#BB00BB'>layer_details</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>set_bias_learning_rate_multiplier</font><font face='Lucida Console'>(</font><font color='#979000'>300</font><font face='Lucida Console'>)</font>;
        sgd <font color='#BB00BB'>defsolver</font><font face='Lucida Console'>(</font><font color='#979000'>0</font>,<font color='#979000'>0.9</font><font face='Lucida Console'>)</font>;
        dnn_trainer<font color='#5555FF'>&lt;</font>net_type<font color='#5555FF'>&gt;</font> <font color='#BB00BB'>trainer</font><font face='Lucida Console'>(</font>net, defsolver<font face='Lucida Console'>)</font>;
        trainer.<font color='#BB00BB'>set_learning_rate</font><font face='Lucida Console'>(</font><font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;
        trainer.<font color='#BB00BB'>set_min_learning_rate</font><font face='Lucida Console'>(</font><font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>8</font><font face='Lucida Console'>)</font>;
        trainer.<font color='#BB00BB'>set_mini_batch_size</font><font face='Lucida Console'>(</font><font color='#979000'>50</font><font face='Lucida Console'>)</font>;
        trainer.<font color='#BB00BB'>set_max_num_epochs</font><font face='Lucida Console'>(</font><font color='#979000'>570</font><font face='Lucida Console'>)</font>;
        trainer.<font color='#BB00BB'>train</font><font face='Lucida Console'>(</font>x, y<font face='Lucida Console'>)</font>;

        <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> slope <font color='#5555FF'>=</font> layer<font color='#5555FF'>&lt;</font><font color='#979000'>1</font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font>.<font color='#BB00BB'>layer_details</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_weights</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[<font color='#979000'>0</font>];
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> slope_error <font color='#5555FF'>=</font> <font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>true_slope <font color='#5555FF'>-</font> slope<font face='Lucida Console'>)</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> intercept <font color='#5555FF'>=</font> layer<font color='#5555FF'>&lt;</font><font color='#979000'>1</font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font>.<font color='#BB00BB'>layer_details</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_biases</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[<font color='#979000'>0</font>];
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> intercept_error <font color='#5555FF'>=</font> <font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>true_intercept <font color='#5555FF'>-</font> intercept<font face='Lucida Console'>)</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> eps_slope <font color='#5555FF'>=</font> <font color='#979000'>0.01</font>, eps_intercept <font color='#5555FF'>=</font> <font color='#979000'>0.1</font>;

        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>slope_error: </font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> slope_error;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>intercept_error: </font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> intercept_error;
        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>slope_error <font color='#5555FF'>&lt;</font><font color='#5555FF'>=</font> eps_slope,
                      "<font color='#CC0000'>Expected slope = </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> true_slope <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> Estimated slope = </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> slope <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> Error limit = </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> eps_slope<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>intercept_error <font color='#5555FF'>&lt;</font><font color='#5555FF'>=</font> eps_intercept,
                      "<font color='#CC0000'>Expected intercept = </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> true_intercept <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> Estimated intercept = </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> intercept <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> Error limit = </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> eps_intercept<font face='Lucida Console'>)</font>;

    <b>}</b>

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'><u>void</u></font> <b><a name='test_simple_linear_regression_with_mult_prev'></a>test_simple_linear_regression_with_mult_prev</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#BB00BB'>srand</font><font face='Lucida Console'>(</font><font color='#979000'>1234</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>int</u></font> num_samples <font color='#5555FF'>=</font> <font color='#979000'>1000</font>;
        ::std::vector<font color='#5555FF'>&lt;</font>matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font> <font color='#BB00BB'>x</font><font face='Lucida Console'>(</font>num_samples<font face='Lucida Console'>)</font>;
        ::std::vector<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font> <font color='#BB00BB'>y</font><font face='Lucida Console'>(</font>num_samples<font face='Lucida Console'>)</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> true_slope <font color='#5555FF'>=</font> <font color='#979000'>2.0</font>;
        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font> <font color='#0000FF'><u>int</u></font> ii <font color='#5555FF'>=</font> <font color='#979000'>0</font>; ii <font color='#5555FF'>&lt;</font> num_samples; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>ii <font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> val <font color='#5555FF'>=</font> <font color='#0000FF'>static_cast</font><font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>ii<font color='#5555FF'>-</font><font color='#979000'>500</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font color='#979000'>100</font>;
            matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font><font color='#5555FF'>&gt;</font> <font color='#BB00BB'>tmp</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,<font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            tmp <font color='#5555FF'>=</font> val;
            x[ii] <font color='#5555FF'>=</font> tmp;
            y[ii] <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font> true_slope<font color='#5555FF'>*</font><font color='#0000FF'>static_cast</font><font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>val<font color='#5555FF'>*</font>val<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#BB00BB'>randomize_samples</font><font face='Lucida Console'>(</font>x,y<font face='Lucida Console'>)</font>;

        <font color='#0000FF'>using</font> net_type <font color='#5555FF'>=</font> loss_mean_squared<font color='#5555FF'>&lt;</font>fc<font color='#5555FF'>&lt;</font><font color='#979000'>1</font>, mult_prev1<font color='#5555FF'>&lt;</font>fc<font color='#5555FF'>&lt;</font><font color='#979000'>2</font>,tag1<font color='#5555FF'>&lt;</font>fc<font color='#5555FF'>&lt;</font><font color='#979000'>2</font>,input<font color='#5555FF'>&lt;</font>matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>;
        net_type net;
        sgd <font color='#BB00BB'>defsolver</font><font face='Lucida Console'>(</font><font color='#979000'>0</font>,<font color='#979000'>0.9</font><font face='Lucida Console'>)</font>;
        dnn_trainer<font color='#5555FF'>&lt;</font>net_type<font color='#5555FF'>&gt;</font> <font color='#BB00BB'>trainer</font><font face='Lucida Console'>(</font>net, defsolver<font face='Lucida Console'>)</font>;
        trainer.<font color='#BB00BB'>set_learning_rate</font><font face='Lucida Console'>(</font><font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;
        trainer.<font color='#BB00BB'>set_min_learning_rate</font><font face='Lucida Console'>(</font><font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>11</font><font face='Lucida Console'>)</font>;
        trainer.<font color='#BB00BB'>set_mini_batch_size</font><font face='Lucida Console'>(</font><font color='#979000'>50</font><font face='Lucida Console'>)</font>;
        trainer.<font color='#BB00BB'>set_max_num_epochs</font><font face='Lucida Console'>(</font><font color='#979000'>2000</font><font face='Lucida Console'>)</font>;
        trainer.<font color='#BB00BB'>train</font><font face='Lucida Console'>(</font>x, y<font face='Lucida Console'>)</font>;

        running_stats<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font><font color='#5555FF'>&gt;</font> rs;
        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> x.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'><u>double</u></font> val <font color='#5555FF'>=</font> y[i];
            <font color='#0000FF'><u>double</u></font> out <font color='#5555FF'>=</font> <font color='#BB00BB'>net</font><font face='Lucida Console'>(</font>x[i]<font face='Lucida Console'>)</font>;
            rs.<font color='#BB00BB'>add</font><font face='Lucida Console'>(</font>std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>val<font color='#5555FF'>-</font>out<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <b>}</b>
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>rs.mean(): </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> rs.<font color='#BB00BB'>mean</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>rs.stddev(): </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> rs.<font color='#BB00BB'>stddev</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>rs.max(): </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> rs.<font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>rs.<font color='#BB00BB'>mean</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>0.1</font><font face='Lucida Console'>)</font>;
    <b>}</b>

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'><u>void</u></font> <b><a name='test_multioutput_linear_regression'></a>test_multioutput_linear_regression</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>int</u></font> num_outputs <font color='#5555FF'>=</font> <font color='#979000'>2</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>int</u></font> num_samples <font color='#5555FF'>=</font> <font color='#979000'>1000</font>;
        ::std::vector<font color='#5555FF'>&lt;</font>matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font> <font color='#BB00BB'>x</font><font face='Lucida Console'>(</font>num_samples<font face='Lucida Console'>)</font>;
        ::std::vector<font color='#5555FF'>&lt;</font>matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font> <font color='#BB00BB'>y</font><font face='Lucida Console'>(</font>num_samples<font face='Lucida Console'>)</font>;
        ::std::default_random_engine <font color='#BB00BB'>generator</font><font face='Lucida Console'>(</font><font color='#979000'>16</font><font face='Lucida Console'>)</font>;
        ::std::normal_distribution<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font> <font color='#BB00BB'>distribution</font><font face='Lucida Console'>(</font><font color='#979000'>0</font>,<font color='#979000'>0.1</font><font face='Lucida Console'>)</font>;
        ::std::normal_distribution<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font> <font color='#BB00BB'>slope_distribution</font><font face='Lucida Console'>(</font><font color='#979000'>10</font>,<font color='#979000'>5</font><font face='Lucida Console'>)</font>;
        ::std::normal_distribution<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font> <font color='#BB00BB'>intercept_distribution</font><font face='Lucida Console'>(</font><font color='#979000'>50</font>,<font color='#979000'>10</font><font face='Lucida Console'>)</font>;
        ::std::vector<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font> <font color='#BB00BB'>true_intercepts</font><font face='Lucida Console'>(</font>num_outputs<font face='Lucida Console'>)</font>;
        ::std::vector<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font> <font color='#BB00BB'>true_slopes</font><font face='Lucida Console'>(</font>num_outputs<font face='Lucida Console'>)</font>;
        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font> <font color='#0000FF'><u>int</u></font> jj <font color='#5555FF'>=</font> <font color='#979000'>0</font>; jj <font color='#5555FF'>&lt;</font> num_outputs; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>jj <font face='Lucida Console'>)</font>
        <b>{</b>
            true_slopes[jj] <font color='#5555FF'>=</font> <font color='#BB00BB'>slope_distribution</font><font face='Lucida Console'>(</font>generator<font face='Lucida Console'>)</font>;
            true_intercepts[jj] <font color='#5555FF'>=</font> <font color='#BB00BB'>intercept_distribution</font><font face='Lucida Console'>(</font>generator<font face='Lucida Console'>)</font>;
        <b>}</b>
        matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font> <font color='#BB00BB'>ytmp</font><font face='Lucida Console'>(</font>num_outputs, <font color='#979000'>1</font><font face='Lucida Console'>)</font>;
        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font> <font color='#0000FF'><u>int</u></font> ii <font color='#5555FF'>=</font> <font color='#979000'>0</font>; ii <font color='#5555FF'>&lt;</font> num_samples; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>ii <font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> val <font color='#5555FF'>=</font> <font color='#0000FF'>static_cast</font><font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>ii<font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font color='#979000'>10</font>;
            matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font><font color='#5555FF'>&gt;</font> <font color='#BB00BB'>tmp</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,<font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            tmp <font color='#5555FF'>=</font> val;
            x[ii] <font color='#5555FF'>=</font> tmp;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font> <font color='#0000FF'><u>int</u></font> jj <font color='#5555FF'>=</font> <font color='#979000'>0</font>; jj <font color='#5555FF'>&lt;</font> num_outputs; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>jj <font face='Lucida Console'>)</font>
                <font color='#BB00BB'>ytmp</font><font face='Lucida Console'>(</font>jj, <font color='#979000'>0</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font>true_intercepts[jj] <font color='#5555FF'>+</font> true_slopes[jj]<font color='#5555FF'>*</font><font color='#0000FF'>static_cast</font><font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>val<font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> <font color='#BB00BB'>distribution</font><font face='Lucida Console'>(</font>generator<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

            y[ii] <font color='#5555FF'>=</font> ytmp;
        <b>}</b>

        <font color='#0000FF'>using</font> net_type <font color='#5555FF'>=</font> loss_mean_squared_multioutput<font color='#5555FF'>&lt;</font>fc<font color='#5555FF'>&lt;</font>num_outputs, input<font color='#5555FF'>&lt;</font>matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>;
        net_type net;
        layer<font color='#5555FF'>&lt;</font><font color='#979000'>1</font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font>.<font color='#BB00BB'>layer_details</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>set_bias_learning_rate_multiplier</font><font face='Lucida Console'>(</font><font color='#979000'>900</font><font face='Lucida Console'>)</font>;
        sgd <font color='#BB00BB'>defsolver</font><font face='Lucida Console'>(</font><font color='#979000'>0</font>,<font color='#979000'>0.9</font><font face='Lucida Console'>)</font>;
        dnn_trainer<font color='#5555FF'>&lt;</font>net_type<font color='#5555FF'>&gt;</font> <font color='#BB00BB'>trainer</font><font face='Lucida Console'>(</font>net, defsolver<font face='Lucida Console'>)</font>;
        trainer.<font color='#BB00BB'>set_learning_rate</font><font face='Lucida Console'>(</font><font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;
        trainer.<font color='#BB00BB'>set_min_learning_rate</font><font face='Lucida Console'>(</font><font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font><font face='Lucida Console'>)</font>;
        trainer.<font color='#BB00BB'>set_mini_batch_size</font><font face='Lucida Console'>(</font><font color='#979000'>50</font><font face='Lucida Console'>)</font>;
        trainer.<font color='#BB00BB'>set_max_num_epochs</font><font face='Lucida Console'>(</font><font color='#979000'>170</font><font face='Lucida Console'>)</font>;
        trainer.<font color='#BB00BB'>train</font><font face='Lucida Console'>(</font>x, y<font face='Lucida Console'>)</font>;

        <font color='#0000FF'><u>float</u></font> slope_error <font color='#5555FF'>=</font> <font color='#979000'>0.0</font>;
        <font color='#0000FF'><u>float</u></font> intercept_error <font color='#5555FF'>=</font> <font color='#979000'>0.0</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> eps_slope <font color='#5555FF'>=</font> <font color='#979000'>0.05</font>, eps_intercept <font color='#5555FF'>=</font> <font color='#979000'>0.1</font>;

        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font> <font color='#0000FF'><u>int</u></font> jj <font color='#5555FF'>=</font> <font color='#979000'>0</font>; jj <font color='#5555FF'>&lt;</font> num_outputs; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>jj <font face='Lucida Console'>)</font>
        <b>{</b>
            slope_error <font color='#5555FF'>+</font><font color='#5555FF'>=</font> <font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>layer<font color='#5555FF'>&lt;</font><font color='#979000'>1</font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font>.<font color='#BB00BB'>layer_details</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_weights</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[jj] <font color='#5555FF'>-</font> true_slopes[jj]<font face='Lucida Console'>)</font>;
            intercept_error <font color='#5555FF'>+</font><font color='#5555FF'>=</font> <font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>layer<font color='#5555FF'>&lt;</font><font color='#979000'>1</font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font>.<font color='#BB00BB'>layer_details</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_biases</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[jj] <font color='#5555FF'>-</font> true_intercepts[jj]<font face='Lucida Console'>)</font>;
        <b>}</b>

        slope_error <font color='#5555FF'>/</font><font color='#5555FF'>=</font> <font color='#0000FF'><u>float</u></font><font face='Lucida Console'>(</font>num_outputs<font face='Lucida Console'>)</font>;
        intercept_error <font color='#5555FF'>/</font><font color='#5555FF'>=</font> <font color='#0000FF'><u>float</u></font><font face='Lucida Console'>(</font>num_outputs<font face='Lucida Console'>)</font>;

        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>slope_error <font color='#5555FF'>&lt;</font><font color='#5555FF'>=</font> eps_slope,
                      "<font color='#CC0000'>Average absolute slope error = </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> slope_error <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> Error limit = </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> eps_slope<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>intercept_error <font color='#5555FF'>&lt;</font><font color='#5555FF'>=</font> eps_intercept,
                      "<font color='#CC0000'>Average absolute intercept error = </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> intercept_error <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> Error limit = </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> eps_intercept<font face='Lucida Console'>)</font>;

    <b>}</b>

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'><u>void</u></font> <b><a name='test_simple_autoencoder'></a>test_simple_autoencoder</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

        <font color='#BB00BB'>srand</font><font face='Lucida Console'>(</font><font color='#979000'>1234</font><font face='Lucida Console'>)</font>;

        <font color='#0000FF'>const</font> <font color='#0000FF'><u>int</u></font> output_width <font color='#5555FF'>=</font> <font color='#979000'>7</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>int</u></font> output_height <font color='#5555FF'>=</font> <font color='#979000'>7</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>int</u></font> num_samples <font color='#5555FF'>=</font> <font color='#979000'>100</font>;
        ::std::vector<font color='#5555FF'>&lt;</font>matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font> <font color='#BB00BB'>x</font><font face='Lucida Console'>(</font>num_samples<font face='Lucida Console'>)</font>;

        matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font> <font color='#BB00BB'>tmp</font><font face='Lucida Console'>(</font>output_width, output_height<font face='Lucida Console'>)</font>;
        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>int</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> num_samples; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>int</u></font> model <font color='#5555FF'>=</font> i <font color='#5555FF'>%</font> <font color='#979000'>4</font>;

            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>int</u></font> r <font color='#5555FF'>=</font> <font color='#979000'>0</font>; r <font color='#5555FF'>&lt;</font> output_height; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>r<font face='Lucida Console'>)</font>
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>int</u></font> c <font color='#5555FF'>=</font> <font color='#979000'>0</font>; c <font color='#5555FF'>&lt;</font> output_width; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>c<font face='Lucida Console'>)</font>
                    <font color='#0000FF'>switch</font> <font face='Lucida Console'>(</font>model<font face='Lucida Console'>)</font> <b>{</b>
                    <font color='#0000FF'>case</font> <font color='#979000'>0</font>: <font color='#BB00BB'>tmp</font><font face='Lucida Console'>(</font>r, c<font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> r <font color='#5555FF'>/</font> output_height; <font color='#0000FF'>break</font>;
                    <font color='#0000FF'>case</font> <font color='#979000'>1</font>: <font color='#BB00BB'>tmp</font><font face='Lucida Console'>(</font>r, c<font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> c <font color='#5555FF'>/</font> output_width; <font color='#0000FF'>break</font>;
                    <font color='#0000FF'>case</font> <font color='#979000'>2</font>: <font color='#BB00BB'>tmp</font><font face='Lucida Console'>(</font>r, c<font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>1.0</font> <font color='#5555FF'>-</font> r <font color='#5555FF'>/</font> output_height; <font color='#0000FF'>break</font>;
                    <font color='#0000FF'>case</font> <font color='#979000'>3</font>: <font color='#BB00BB'>tmp</font><font face='Lucida Console'>(</font>r, c<font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>1.0</font> <font color='#5555FF'>-</font> c <font color='#5555FF'>/</font> output_width; <font color='#0000FF'>break</font>;
                    <font color='#0000FF'>default</font>: <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, "<font color='#CC0000'>Invalid model: </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> model <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> (should be between 0 and 3)</font>"<font face='Lucida Console'>)</font>;
                    <b>}</b>

            x[i] <font color='#5555FF'>=</font> tmp;
        <b>}</b>

        <font color='#0000FF'>using</font> net_type <font color='#5555FF'>=</font> loss_mean_squared_per_pixel<font color='#5555FF'>&lt;</font>
                            cont<font color='#5555FF'>&lt;</font><font color='#979000'>1</font>,output_height,output_width,<font color='#979000'>2</font>,<font color='#979000'>2</font>,
                            relu<font color='#5555FF'>&lt;</font>con<font color='#5555FF'>&lt;</font><font color='#979000'>4</font>,output_height,output_width,<font color='#979000'>2</font>,<font color='#979000'>2</font>,
                            input<font color='#5555FF'>&lt;</font>matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>;
        net_type net;

        <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> autoencoder_error <font color='#5555FF'>=</font> [<font color='#5555FF'>&amp;</font>x, <font color='#5555FF'>&amp;</font>net, <font color='#5555FF'>&amp;</font>output_height, <font color='#5555FF'>&amp;</font>output_width]<font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> y <font color='#5555FF'>=</font> <font color='#BB00BB'>net</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font>;
            <font color='#0000FF'><u>double</u></font> error <font color='#5555FF'>=</font> <font color='#979000'>0.0</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> x.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>int</u></font> r <font color='#5555FF'>=</font> <font color='#979000'>0</font>; r <font color='#5555FF'>&lt;</font> output_height; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>r<font face='Lucida Console'>)</font>
                    <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>int</u></font> c <font color='#5555FF'>=</font> <font color='#979000'>0</font>; c <font color='#5555FF'>&lt;</font> output_width; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>c<font face='Lucida Console'>)</font>
                        error <font color='#5555FF'>+</font><font color='#5555FF'>=</font> <font color='#BB00BB'>fabs</font><font face='Lucida Console'>(</font>y[i]<font face='Lucida Console'>(</font>r, c<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> x[i]<font face='Lucida Console'>(</font>r, c<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

            <font color='#0000FF'>return</font> error <font color='#5555FF'>/</font> <font face='Lucida Console'>(</font>x.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>*</font> output_height <font color='#5555FF'>*</font> output_width<font face='Lucida Console'>)</font>;
        <b>}</b>;

        <font color='#009900'>// The autoencoder can't be very good before it's been trained
</font>        <font color='#009900'>// (or at least the probability of the reconstruction error
</font>        <font color='#009900'>// being small should be super low; in fact, the error ought to
</font>        <font color='#009900'>// be much higher than 0.01, however since the initialization
</font>        <font color='#009900'>// is random, putting the limit below too high could make the
</font>        <font color='#009900'>// tests fail when other, unrelated tests are added into the
</font>        <font color='#009900'>// sequence)
</font>        <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> error_before <font color='#5555FF'>=</font> <font color='#BB00BB'>autoencoder_error</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>error_before <font color='#5555FF'>&gt;</font> <font color='#979000'>0.01</font>, "<font color='#CC0000'>Autoencoder error before training = </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> error_before<font face='Lucida Console'>)</font>;

        <font color='#009900'>// Make sure there's an information bottleneck, as intended
</font>        <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font> output2 <font color='#5555FF'>=</font> dlib::layer<font color='#5555FF'>&lt;</font><font color='#979000'>2</font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>output2.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>output2.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>output2.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>4</font><font face='Lucida Console'>)</font>;

        sgd <font color='#BB00BB'>defsolver</font><font face='Lucida Console'>(</font><font color='#979000'>0</font>,<font color='#979000'>0.9</font><font face='Lucida Console'>)</font>;
        dnn_trainer<font color='#5555FF'>&lt;</font>net_type<font color='#5555FF'>&gt;</font> <font color='#BB00BB'>trainer</font><font face='Lucida Console'>(</font>net, defsolver<font face='Lucida Console'>)</font>;
        trainer.<font color='#BB00BB'>set_learning_rate</font><font face='Lucida Console'>(</font><font color='#979000'>0.01</font><font face='Lucida Console'>)</font>;
        trainer.<font color='#BB00BB'>set_max_num_epochs</font><font face='Lucida Console'>(</font><font color='#979000'>1000</font><font face='Lucida Console'>)</font>;
        trainer.<font color='#BB00BB'>train</font><font face='Lucida Console'>(</font>x, x<font face='Lucida Console'>)</font>;

        <font color='#009900'>// Now we should have learned everything there is to it
</font>        <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> error_after <font color='#5555FF'>=</font> <font color='#BB00BB'>autoencoder_error</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>error_after <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font>, "<font color='#CC0000'>Autoencoder error after training = </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> error_after<font face='Lucida Console'>)</font>;
    <b>}</b>

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'><u>void</u></font> <b><a name='test_loss_mean_squared_per_channel_and_pixel'></a>test_loss_mean_squared_per_channel_and_pixel</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

        <font color='#0000FF'>const</font> <font color='#0000FF'><u>int</u></font> num_samples <font color='#5555FF'>=</font> <font color='#979000'>1000</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> num_channels <font color='#5555FF'>=</font> <font color='#979000'>10</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> dimension <font color='#5555FF'>=</font> <font color='#979000'>3</font>;
        ::std::vector<font color='#5555FF'>&lt;</font>matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font> inputs;
        ::std::vector<font color='#5555FF'>&lt;</font>::std::array<font color='#5555FF'>&lt;</font>matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font>, num_channels<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font> labels;
        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>int</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> num_samples; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
        <b>{</b>
            matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font> x <font color='#5555FF'>=</font> matrix_cast<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font><font color='#BB00BB'>randm</font><font face='Lucida Console'>(</font><font color='#979000'>5</font>, dimension<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font> w <font color='#5555FF'>=</font> matrix_cast<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font><font color='#BB00BB'>randm</font><font face='Lucida Console'>(</font>num_channels, <font color='#979000'>5</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font> y <font color='#5555FF'>=</font> w <font color='#5555FF'>*</font> x;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>y.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> num_channels<font face='Lucida Console'>)</font>;
            ::std::array<font color='#5555FF'>&lt;</font>matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font>, num_channels<font color='#5555FF'>&gt;</font> y_arr;
            <font color='#009900'>// convert y to an array of matrices
</font>            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> c <font color='#5555FF'>=</font> <font color='#979000'>0</font>; c <font color='#5555FF'>&lt;</font> num_channels; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>c<font face='Lucida Console'>)</font>
            <b>{</b>
                y_arr[c] <font color='#5555FF'>=</font> <font color='#BB00BB'>rowm</font><font face='Lucida Console'>(</font>y, c<font face='Lucida Console'>)</font>;
            <b>}</b>
            inputs.<font color='#BB00BB'>push_back</font><font face='Lucida Console'>(</font>::std::<font color='#BB00BB'>move</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            labels.<font color='#BB00BB'>push_back</font><font face='Lucida Console'>(</font>::std::<font color='#BB00BB'>move</font><font face='Lucida Console'>(</font>y_arr<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> num_outputs <font color='#5555FF'>=</font> num_channels <font color='#5555FF'>*</font> dimension;
        <font color='#0000FF'>using</font> net_type <font color='#5555FF'>=</font> loss_mean_squared_per_channel_and_pixel<font color='#5555FF'>&lt;</font>num_channels,
                            extract<font color='#5555FF'>&lt;</font><font color='#979000'>0</font>, num_channels, <font color='#979000'>1</font>, dimension,
                            fc<font color='#5555FF'>&lt;</font>num_outputs,
                            relu<font color='#5555FF'>&lt;</font>bn_fc<font color='#5555FF'>&lt;</font>fc<font color='#5555FF'>&lt;</font><font color='#979000'>500</font>,
                            input<font color='#5555FF'>&lt;</font>matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>;
        net_type net;

        <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> compute_error <font color='#5555FF'>=</font> [<font color='#5555FF'>&amp;</font>inputs, <font color='#5555FF'>&amp;</font>labels, <font color='#5555FF'>&amp;</font>net, num_channels]<font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> out <font color='#5555FF'>=</font> <font color='#BB00BB'>net</font><font face='Lucida Console'>(</font>inputs<font face='Lucida Console'>)</font>;
            <font color='#0000FF'><u>double</u></font> error <font color='#5555FF'>=</font> <font color='#979000'>0.0</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> out.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> c <font color='#5555FF'>=</font> <font color='#979000'>0</font>; c <font color='#5555FF'>&lt;</font> num_channels; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>c<font face='Lucida Console'>)</font>
                <b>{</b>
                    error <font color='#5555FF'>+</font><font color='#5555FF'>=</font> <font color='#BB00BB'>mean</font><font face='Lucida Console'>(</font><font color='#BB00BB'>squared</font><font face='Lucida Console'>(</font>out[i][c] <font color='#5555FF'>-</font> labels[i][c]<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                <b>}</b>
            <b>}</b>
            <font color='#0000FF'>return</font> error <font color='#5555FF'>/</font> out.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>/</font> num_channels;
        <b>}</b>;

        <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> error_before <font color='#5555FF'>=</font> <font color='#BB00BB'>compute_error</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        dnn_trainer<font color='#5555FF'>&lt;</font>net_type<font color='#5555FF'>&gt;</font> <font color='#BB00BB'>trainer</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font>;
        trainer.<font color='#BB00BB'>set_learning_rate</font><font face='Lucida Console'>(</font><font color='#979000'>0.1</font><font face='Lucida Console'>)</font>;
        trainer.<font color='#BB00BB'>set_iterations_without_progress_threshold</font><font face='Lucida Console'>(</font><font color='#979000'>500</font><font face='Lucida Console'>)</font>;
        trainer.<font color='#BB00BB'>set_min_learning_rate</font><font face='Lucida Console'>(</font><font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font><font face='Lucida Console'>)</font>;
        trainer.<font color='#BB00BB'>set_mini_batch_size</font><font face='Lucida Console'>(</font><font color='#979000'>50</font><font face='Lucida Console'>)</font>;
        trainer.<font color='#BB00BB'>set_max_num_epochs</font><font face='Lucida Console'>(</font><font color='#979000'>100</font><font face='Lucida Console'>)</font>;
        trainer.<font color='#BB00BB'>train</font><font face='Lucida Console'>(</font>inputs, labels<font face='Lucida Console'>)</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> error_after <font color='#5555FF'>=</font> <font color='#BB00BB'>compute_error</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>error_after <font color='#5555FF'>&lt;</font> error_before, "<font color='#CC0000'>multi channel error increased after training</font>"<font face='Lucida Console'>)</font>;
<font color='#0000FF'>#if</font> DLIB_USE_CUDA
        cuda::compute_loss_mean_squared_per_channel_and_pixel cuda_compute;
        cpu::compute_loss_mean_squared_per_channel_and_pixel cpu_compute;
        <font color='#0000FF'><u>double</u></font> cuda_loss, cpu_loss;
        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> output_tensor <font color='#5555FF'>=</font> net.<font color='#BB00BB'>subnet</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        resizable_tensor <font color='#BB00BB'>cuda_grad</font><font face='Lucida Console'>(</font>output_tensor<font face='Lucida Console'>)</font>, <font color='#BB00BB'>cpu_grad</font><font face='Lucida Console'>(</font>output_tensor<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>cuda_compute</font><font face='Lucida Console'>(</font>labels.<font color='#BB00BB'>begin</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, output_tensor, cuda_grad, cuda_loss<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>cpu_compute</font><font face='Lucida Console'>(</font>labels.<font color='#BB00BB'>begin</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, output_tensor, cpu_grad, cpu_loss<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>cuda_grad.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> cpu_grad.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> cuda_grad.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>::std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#5555FF'>*</font><font face='Lucida Console'>(</font>cuda_grad.<font color='#BB00BB'>begin</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> i<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> <font color='#5555FF'>*</font><font face='Lucida Console'>(</font>cpu_grad.<font color='#BB00BB'>begin</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> i<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>8</font><font face='Lucida Console'>)</font>;
        <b>}</b>
        <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> err <font color='#5555FF'>=</font> <font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>cuda_loss <font color='#5555FF'>-</font> cpu_loss<font face='Lucida Console'>)</font> <font color='#5555FF'>/</font> cpu_loss;
        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>err <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font>, "<font color='#CC0000'>multi channel cuda and cpu losses differ</font>"<font face='Lucida Console'>)</font>;
<font color='#0000FF'>#endif</font>
    <b>}</b>

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'><u>void</u></font> <b><a name='test_loss_binary_log_per_pixel_learned_params_on_trivial_two_pixel_task'></a>test_loss_binary_log_per_pixel_learned_params_on_trivial_two_pixel_task</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

        ::std::vector<font color='#5555FF'>&lt;</font>matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font> <font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><b>{</b> matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font>,<font color='#979000'>2</font>,<font color='#979000'>1</font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font><b>{</b> <font color='#5555FF'>-</font><font color='#979000'>1</font>, <font color='#979000'>1</font> <b>}</b><font face='Lucida Console'>)</font> <b>}</b><font face='Lucida Console'>)</font>;
        ::std::vector<font color='#5555FF'>&lt;</font>matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font> <font color='#BB00BB'>y</font><font face='Lucida Console'>(</font><b>{</b> matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font>,<font color='#979000'>2</font>,<font color='#979000'>1</font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font><b>{</b> <font color='#5555FF'>-</font><font color='#979000'>1</font>, <font color='#979000'>1</font> <b>}</b><font face='Lucida Console'>)</font> <b>}</b><font face='Lucida Console'>)</font>;

        <font color='#0000FF'>using</font> net_type <font color='#5555FF'>=</font> loss_binary_log_per_pixel<font color='#5555FF'>&lt;</font>con<font color='#5555FF'>&lt;</font><font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,input<font color='#5555FF'>&lt;</font>matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>;
        net_type net;

        dnn_trainer<font color='#5555FF'>&lt;</font>net_type<font color='#5555FF'>&gt;</font> <font color='#BB00BB'>trainer</font><font face='Lucida Console'>(</font>net, <font color='#BB00BB'>sgd</font><font face='Lucida Console'>(</font><font color='#979000'>0</font>,<font color='#979000'>0</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        trainer.<font color='#BB00BB'>set_learning_rate</font><font face='Lucida Console'>(</font><font color='#979000'>1e7</font><font face='Lucida Console'>)</font>;
        trainer.<font color='#BB00BB'>set_max_num_epochs</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>;
        trainer.<font color='#BB00BB'>train</font><font face='Lucida Console'>(</font>x, y<font face='Lucida Console'>)</font>;

        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> learned_params <font color='#5555FF'>=</font> layer<font color='#5555FF'>&lt;</font><font color='#979000'>1</font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font>.<font color='#BB00BB'>layer_details</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_layer_params</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> learned_params_data <font color='#5555FF'>=</font> learned_params.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>learned_params_data[<font color='#979000'>0</font>] <font color='#5555FF'>&gt;</font> <font color='#979000'>1e5</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>learned_params_data[<font color='#979000'>1</font>]<font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;
    <b>}</b>

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'><u>void</u></font> <b><a name='test_loss_binary_log_per_pixel_outputs_on_trivial_task'></a>test_loss_binary_log_per_pixel_outputs_on_trivial_task</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

        constexpr <font color='#0000FF'><u>int</u></font> input_height <font color='#5555FF'>=</font> <font color='#979000'>7</font>;
        constexpr <font color='#0000FF'><u>int</u></font> input_width <font color='#5555FF'>=</font> <font color='#979000'>5</font>;
        constexpr <font color='#0000FF'><u>int</u></font> output_height <font color='#5555FF'>=</font> input_height;
        constexpr <font color='#0000FF'><u>int</u></font> output_width <font color='#5555FF'>=</font> input_width;
        constexpr <font color='#0000FF'><u>int</u></font> num_samples <font color='#5555FF'>=</font> <font color='#979000'>7</font>;

        ::std::vector<font color='#5555FF'>&lt;</font>matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font> <font color='#BB00BB'>x</font><font face='Lucida Console'>(</font>num_samples<font face='Lucida Console'>)</font>;
        ::std::vector<font color='#5555FF'>&lt;</font>matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font> <font color='#BB00BB'>y</font><font face='Lucida Console'>(</font>num_samples<font face='Lucida Console'>)</font>;

        matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font><font color='#5555FF'>&gt;</font> <font color='#BB00BB'>xtmp</font><font face='Lucida Console'>(</font>input_height, input_width<font face='Lucida Console'>)</font>;
        matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font> <font color='#BB00BB'>ytmp</font><font face='Lucida Console'>(</font>output_height, output_width<font face='Lucida Console'>)</font>;

        ::std::default_random_engine <font color='#BB00BB'>generator</font><font face='Lucida Console'>(</font><font color='#979000'>16</font><font face='Lucida Console'>)</font>;
        ::std::normal_distribution<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font><font color='#5555FF'>&gt;</font> <font color='#BB00BB'>n01</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font>;

        <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> z <font color='#5555FF'>=</font> <font color='#979000'>0.674490</font>; <font color='#009900'>// This should give us a 50/50 split between the classes
</font>
        <font color='#009900'>// Generate training data: random inputs x, and the corresponding target outputs y
</font>        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>int</u></font> ii <font color='#5555FF'>=</font> <font color='#979000'>0</font>; ii <font color='#5555FF'>&lt;</font> num_samples; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>ii<font face='Lucida Console'>)</font> <b>{</b>
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>int</u></font> jj <font color='#5555FF'>=</font> <font color='#979000'>0</font>; jj <font color='#5555FF'>&lt;</font> input_height; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>jj<font face='Lucida Console'>)</font> <b>{</b>
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>int</u></font> kk <font color='#5555FF'>=</font> <font color='#979000'>0</font>; kk <font color='#5555FF'>&lt;</font> input_width; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>kk<font face='Lucida Console'>)</font> <b>{</b>
                    <font color='#BB00BB'>xtmp</font><font face='Lucida Console'>(</font>jj, kk<font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#BB00BB'>n01</font><font face='Lucida Console'>(</font>generator<font face='Lucida Console'>)</font>;
                    <font color='#BB00BB'>ytmp</font><font face='Lucida Console'>(</font>jj, kk<font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>xtmp</font><font face='Lucida Console'>(</font>jj, kk<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&gt;</font> z ? <font color='#979000'>1.f</font> : <font color='#5555FF'>-</font><font color='#979000'>1.f</font>;
                <b>}</b>
            <b>}</b>
            x[ii] <font color='#5555FF'>=</font> xtmp;
            y[ii] <font color='#5555FF'>=</font> ytmp;
        <b>}</b>

        <font color='#0000FF'>using</font> net_type <font color='#5555FF'>=</font> loss_binary_log_per_pixel<font color='#5555FF'>&lt;</font>con<font color='#5555FF'>&lt;</font><font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,relu<font color='#5555FF'>&lt;</font>con<font color='#5555FF'>&lt;</font><font color='#979000'>10</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,input<font color='#5555FF'>&lt;</font>matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>;
        net_type net;

        dnn_trainer<font color='#5555FF'>&lt;</font>net_type<font color='#5555FF'>&gt;</font> <font color='#BB00BB'>trainer</font><font face='Lucida Console'>(</font>net, <font color='#BB00BB'>sgd</font><font face='Lucida Console'>(</font><font color='#979000'>0</font>, <font color='#979000'>0.9</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        trainer.<font color='#BB00BB'>set_learning_rate</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>;
        trainer.<font color='#BB00BB'>set_max_num_epochs</font><font face='Lucida Console'>(</font><font color='#979000'>800</font><font face='Lucida Console'>)</font>;
        trainer.<font color='#BB00BB'>train</font><font face='Lucida Console'>(</font>x, y<font face='Lucida Console'>)</font>;

        <font color='#009900'>// The learning task is easy, so the net should have no problem
</font>        <font color='#009900'>// getting all the outputs right.
</font>        <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> response <font color='#5555FF'>=</font> <font color='#BB00BB'>net</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font>;
        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>int</u></font> ii <font color='#5555FF'>=</font> <font color='#979000'>0</font>; ii <font color='#5555FF'>&lt;</font> num_samples; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>ii<font face='Lucida Console'>)</font>
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>int</u></font> jj <font color='#5555FF'>=</font> <font color='#979000'>0</font>; jj <font color='#5555FF'>&lt;</font> output_height; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>jj<font face='Lucida Console'>)</font>
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>int</u></font> kk <font color='#5555FF'>=</font> <font color='#979000'>0</font>; kk <font color='#5555FF'>&lt;</font> output_width; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>kk<font face='Lucida Console'>)</font>
                    <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font face='Lucida Console'>(</font>response[ii]<font face='Lucida Console'>(</font>jj,kk<font face='Lucida Console'>)</font> <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font face='Lucida Console'>(</font>y[ii]<font face='Lucida Console'>(</font>jj,kk<font face='Lucida Console'>)</font> <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
    <b>}</b>

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'><u>void</u></font> <b><a name='test_loss_binary_log_per_pixel_with_noise_and_pixels_to_ignore'></a>test_loss_binary_log_per_pixel_with_noise_and_pixels_to_ignore</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#009900'>// Test learning when some pixels are to be ignored, etc.
</font>
        <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

        constexpr <font color='#0000FF'><u>int</u></font> input_height <font color='#5555FF'>=</font> <font color='#979000'>5</font>;
        constexpr <font color='#0000FF'><u>int</u></font> input_width <font color='#5555FF'>=</font> <font color='#979000'>7</font>;
        constexpr <font color='#0000FF'><u>int</u></font> output_height <font color='#5555FF'>=</font> input_height;
        constexpr <font color='#0000FF'><u>int</u></font> output_width <font color='#5555FF'>=</font> input_width;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>int</u></font> num_samples <font color='#5555FF'>=</font> <font color='#979000'>1000</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> ignore_probability <font color='#5555FF'>=</font> <font color='#979000'>0.5</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> noise_probability <font color='#5555FF'>=</font> <font color='#979000'>0.05</font>;

        ::std::default_random_engine <font color='#BB00BB'>generator</font><font face='Lucida Console'>(</font><font color='#979000'>16</font><font face='Lucida Console'>)</font>;
        ::std::bernoulli_distribution <font color='#BB00BB'>ignore</font><font face='Lucida Console'>(</font>ignore_probability<font face='Lucida Console'>)</font>;
        ::std::bernoulli_distribution <font color='#BB00BB'>noise_occurrence</font><font face='Lucida Console'>(</font>noise_probability<font face='Lucida Console'>)</font>;
        ::std::bernoulli_distribution <font color='#BB00BB'>noisy_label</font><font face='Lucida Console'>(</font><font color='#979000'>0.5</font><font face='Lucida Console'>)</font>;

        ::std::vector<font color='#5555FF'>&lt;</font>matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font> <font color='#BB00BB'>x</font><font face='Lucida Console'>(</font>num_samples<font face='Lucida Console'>)</font>;
        ::std::vector<font color='#5555FF'>&lt;</font>matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font> <font color='#BB00BB'>y</font><font face='Lucida Console'>(</font>num_samples<font face='Lucida Console'>)</font>;

        ::std::vector<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>int</u></font><font color='#5555FF'>&gt;</font> <font color='#BB00BB'>truth_histogram</font><font face='Lucida Console'>(</font><font color='#979000'>2</font><font face='Lucida Console'>)</font>;

        matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font><font color='#5555FF'>&gt;</font> <font color='#BB00BB'>xtmp</font><font face='Lucida Console'>(</font>input_height, input_width<font face='Lucida Console'>)</font>;
        matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font> <font color='#BB00BB'>ytmp</font><font face='Lucida Console'>(</font>output_height, output_width<font face='Lucida Console'>)</font>;

        <font color='#009900'>// The function to be learned.
</font>        <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> ground_truth <font color='#5555FF'>=</font> []<font face='Lucida Console'>(</font><font color='#0000FF'>const</font> matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&amp;</font> x, <font color='#0000FF'><u>int</u></font> row, <font color='#0000FF'><u>int</u></font> column<font face='Lucida Console'>)</font> <b>{</b>
            <font color='#0000FF'><u>double</u></font> sum <font color='#5555FF'>=</font> <font color='#979000'>0.0</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>int</u></font> first_column <font color='#5555FF'>=</font> std::<font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#979000'>0</font>, column <font color='#5555FF'>-</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>int</u></font> last_column <font color='#5555FF'>=</font> std::<font color='#BB00BB'>min</font><font face='Lucida Console'>(</font><font color='#0000FF'>static_cast</font><font color='#5555FF'>&lt;</font><font color='#0000FF'><u>int</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>x.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>, column <font color='#5555FF'>+</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>int</u></font> c <font color='#5555FF'>=</font> first_column; c <font color='#5555FF'>&lt;</font><font color='#5555FF'>=</font> last_column; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>c<font face='Lucida Console'>)</font> <b>{</b>
                sum <font color='#5555FF'>+</font><font color='#5555FF'>=</font> <font color='#BB00BB'>x</font><font face='Lucida Console'>(</font>row, c<font face='Lucida Console'>)</font>;
            <b>}</b>
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>sum <font color='#5555FF'>&lt;</font> <font color='#979000'>2.0</font> <font color='#5555FF'>*</font> <font face='Lucida Console'>(</font>last_column <font color='#5555FF'>-</font> first_column <font color='#5555FF'>+</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>return</font> sum <font color='#5555FF'>&gt;</font> <font face='Lucida Console'>(</font>last_column <font color='#5555FF'>-</font> first_column <font color='#5555FF'>+</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;
        <b>}</b>;

        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font> <font color='#0000FF'><u>int</u></font> ii <font color='#5555FF'>=</font> <font color='#979000'>0</font>; ii <font color='#5555FF'>&lt;</font> num_samples; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>ii <font face='Lucida Console'>)</font> <b>{</b>
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font> <font color='#0000FF'><u>int</u></font> jj <font color='#5555FF'>=</font> <font color='#979000'>0</font>; jj <font color='#5555FF'>&lt;</font> input_height; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>jj <font face='Lucida Console'>)</font> <b>{</b>
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font> <font color='#0000FF'><u>int</u></font> kk <font color='#5555FF'>=</font> <font color='#979000'>0</font>; kk <font color='#5555FF'>&lt;</font> input_width; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>kk <font face='Lucida Console'>)</font> <b>{</b>
                    <font color='#009900'>// Generate numbers between 0 and 2.
</font>                    <font color='#0000FF'><u>double</u></font> value <font color='#5555FF'>=</font> <font color='#0000FF'>static_cast</font><font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>ii <font color='#5555FF'>+</font> jj <font color='#5555FF'>+</font> kk<font face='Lucida Console'>)</font> <font color='#5555FF'>/</font> <font color='#979000'>10.0</font>;
                    value <font color='#5555FF'>-</font><font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#0000FF'>static_cast</font><font color='#5555FF'>&lt;</font><font color='#0000FF'><u>int</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>value<font face='Lucida Console'>)</font> <font color='#5555FF'>/</font> <font color='#979000'>2</font><font face='Lucida Console'>)</font> <font color='#5555FF'>*</font> <font color='#979000'>2</font>;
                    <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>value <font color='#5555FF'>&gt;</font><font color='#5555FF'>=</font> <font color='#979000'>0.0</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> value <font color='#5555FF'>&lt;</font> <font color='#979000'>2.0</font><font face='Lucida Console'>)</font>;
                    <font color='#BB00BB'>xtmp</font><font face='Lucida Console'>(</font>jj, kk<font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> value;
                <b>}</b>
            <b>}</b>
            x[ii] <font color='#5555FF'>=</font> xtmp;

            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font> <font color='#0000FF'><u>int</u></font> jj <font color='#5555FF'>=</font> <font color='#979000'>0</font>; jj <font color='#5555FF'>&lt;</font> output_height; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>jj <font face='Lucida Console'>)</font> <b>{</b>
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font> <font color='#0000FF'><u>int</u></font> kk <font color='#5555FF'>=</font> <font color='#979000'>0</font>; kk <font color='#5555FF'>&lt;</font> output_width; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>kk <font face='Lucida Console'>)</font> <b>{</b>
                    <font color='#0000FF'>const</font> <font color='#0000FF'><u>bool</u></font> truth <font color='#5555FF'>=</font> <font color='#BB00BB'>ground_truth</font><font face='Lucida Console'>(</font>x[ii], jj, kk<font face='Lucida Console'>)</font>;
                    <font color='#5555FF'>+</font><font color='#5555FF'>+</font>truth_histogram[truth];
                    <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>ignore</font><font face='Lucida Console'>(</font>generator<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <b>{</b>
                        <font color='#BB00BB'>ytmp</font><font face='Lucida Console'>(</font>jj, kk<font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>0.f</font>;
                    <b>}</b>
                    <font color='#0000FF'>else</font> <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>noise_occurrence</font><font face='Lucida Console'>(</font>generator<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <b>{</b>
                        <font color='#BB00BB'>ytmp</font><font face='Lucida Console'>(</font>jj, kk<font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#BB00BB'>noisy_label</font><font face='Lucida Console'>(</font>generator<font face='Lucida Console'>)</font> ? <font color='#979000'>1.f</font> : <font color='#5555FF'>-</font><font color='#979000'>1.f</font>;
                    <b>}</b>
                    <font color='#0000FF'>else</font> <b>{</b>
                        <font color='#BB00BB'>ytmp</font><font face='Lucida Console'>(</font>jj, kk<font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> truth ? <font color='#979000'>1.f</font> : <font color='#5555FF'>-</font><font color='#979000'>1.f</font>;
                    <b>}</b>
                <b>}</b>
            <b>}</b>

            y[ii] <font color='#5555FF'>=</font> ytmp;
        <b>}</b>

        <font color='#0000FF'>const</font> <font color='#0000FF'><u>int</u></font> num_total_elements <font color='#5555FF'>=</font> num_samples <font color='#5555FF'>*</font> output_height <font color='#5555FF'>*</font> output_width;

        <b>{</b> <font color='#009900'>// Require a reasonably balanced truth histogram in order to make sure that a trivial classifier is not enough
</font>            <font color='#0000FF'>const</font> <font color='#0000FF'><u>int</u></font> required_min_histogram_value <font color='#5555FF'>=</font> <font color='#0000FF'>static_cast</font><font color='#5555FF'>&lt;</font><font color='#0000FF'><u>int</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>::std::<font color='#BB00BB'>ceil</font><font face='Lucida Console'>(</font>num_total_elements <font color='#5555FF'>/</font> <font color='#979000'>2.0</font> <font color='#5555FF'>*</font> <font color='#979000'>0.375</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'>auto</font> histogram_value : truth_histogram<font face='Lucida Console'>)</font> <b>{</b>
                <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>histogram_value <font color='#5555FF'>&gt;</font><font color='#5555FF'>=</font> required_min_histogram_value,
                              "<font color='#CC0000'>Histogram value = </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> histogram_value <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>, required = </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> required_min_histogram_value<font face='Lucida Console'>)</font>;
            <b>}</b>
        <b>}</b>

        <font color='#0000FF'>using</font> net_type <font color='#5555FF'>=</font> loss_binary_log_per_pixel<font color='#5555FF'>&lt;</font>con<font color='#5555FF'>&lt;</font><font color='#979000'>1</font>,<font color='#979000'>1</font>,input_width,<font color='#979000'>1</font>,<font color='#979000'>1</font>,input<font color='#5555FF'>&lt;</font>matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>;
        net_type net;
        sgd <font color='#BB00BB'>defsolver</font><font face='Lucida Console'>(</font><font color='#979000'>0</font>,<font color='#979000'>0.9</font><font face='Lucida Console'>)</font>;
        dnn_trainer<font color='#5555FF'>&lt;</font>net_type<font color='#5555FF'>&gt;</font> <font color='#BB00BB'>trainer</font><font face='Lucida Console'>(</font>net, defsolver<font face='Lucida Console'>)</font>;
        trainer.<font color='#BB00BB'>set_learning_rate</font><font face='Lucida Console'>(</font><font color='#979000'>0.1</font><font face='Lucida Console'>)</font>;
        trainer.<font color='#BB00BB'>set_min_learning_rate</font><font face='Lucida Console'>(</font><font color='#979000'>0.01</font><font face='Lucida Console'>)</font>;
        trainer.<font color='#BB00BB'>set_mini_batch_size</font><font face='Lucida Console'>(</font><font color='#979000'>50</font><font face='Lucida Console'>)</font>;
        trainer.<font color='#BB00BB'>set_max_num_epochs</font><font face='Lucida Console'>(</font><font color='#979000'>170</font><font face='Lucida Console'>)</font>;
        trainer.<font color='#BB00BB'>train</font><font face='Lucida Console'>(</font>x, y<font face='Lucida Console'>)</font>;

        <font color='#0000FF'>const</font> ::std::vector<font color='#5555FF'>&lt;</font>matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font> predictions <font color='#5555FF'>=</font> <font color='#BB00BB'>net</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font>;

        <font color='#0000FF'><u>int</u></font> num_correct <font color='#5555FF'>=</font> <font color='#979000'>0</font>;

        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font> <font color='#0000FF'><u>int</u></font> ii <font color='#5555FF'>=</font> <font color='#979000'>0</font>; ii <font color='#5555FF'>&lt;</font> num_samples; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>ii <font face='Lucida Console'>)</font> <b>{</b>
            <font color='#0000FF'>const</font> matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&amp;</font> prediction <font color='#5555FF'>=</font> predictions[ii];
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>prediction.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> output_height<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>prediction.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> output_width<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font> <font color='#0000FF'><u>int</u></font> jj <font color='#5555FF'>=</font> <font color='#979000'>0</font>; jj <font color='#5555FF'>&lt;</font> output_height; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>jj <font face='Lucida Console'>)</font>
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font> <font color='#0000FF'><u>int</u></font> kk <font color='#5555FF'>=</font> <font color='#979000'>0</font>; kk <font color='#5555FF'>&lt;</font> output_width; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>kk <font face='Lucida Console'>)</font>
                    <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>prediction</font><font face='Lucida Console'>(</font>jj, kk<font face='Lucida Console'>)</font> <font color='#5555FF'>&gt;</font> <font color='#979000'>0.f</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#BB00BB'>ground_truth</font><font face='Lucida Console'>(</font>x[ii], jj, kk<font face='Lucida Console'>)</font> <font face='Lucida Console'>)</font>
                        <font color='#5555FF'>+</font><font color='#5555FF'>+</font>num_correct;
        <b>}</b>

        <font color='#009900'>// First some sanity checks.
</font>        <font color='#0000FF'>const</font> <font color='#0000FF'><u>int</u></font> num_correct_max <font color='#5555FF'>=</font> num_total_elements;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>num_correct_max <font color='#5555FF'>=</font><font color='#5555FF'>=</font> ::std::<font color='#BB00BB'>accumulate</font><font face='Lucida Console'>(</font>truth_histogram.<font color='#BB00BB'>begin</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, truth_histogram.<font color='#BB00BB'>end</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, <font color='#979000'>0</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>num_correct <font color='#5555FF'>&lt;</font><font color='#5555FF'>=</font> num_correct_max,
                      "<font color='#CC0000'>Number of correctly classified elements = </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> num_correct <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>, max = </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> num_correct_max<font face='Lucida Console'>)</font>;

        <font color='#009900'>// This is the real test, verifying that we have actually learned something.
</font>        <font color='#0000FF'>const</font> <font color='#0000FF'><u>int</u></font> num_correct_required <font color='#5555FF'>=</font> <font color='#0000FF'>static_cast</font><font color='#5555FF'>&lt;</font><font color='#0000FF'><u>int</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>::std::<font color='#BB00BB'>ceil</font><font face='Lucida Console'>(</font><font color='#979000'>0.9</font> <font color='#5555FF'>*</font> num_correct_max<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>num_correct <font color='#5555FF'>&gt;</font><font color='#5555FF'>=</font> num_correct_required,
                      "<font color='#CC0000'>Number of correctly classified elements = </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> num_correct <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>, required = </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> num_correct_required<font face='Lucida Console'>)</font>;

<font color='#0000FF'>#if</font> DLIB_USE_CUDA
        cuda::compute_loss_binary_log_per_pixel cuda_compute;
        cpu::compute_loss_binary_log_per_pixel cpu_compute;
        <font color='#0000FF'><u>double</u></font> cuda_loss, cpu_loss;
        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> output_tensor <font color='#5555FF'>=</font> net.<font color='#BB00BB'>subnet</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        resizable_tensor <font color='#BB00BB'>cuda_grad</font><font face='Lucida Console'>(</font>output_tensor<font face='Lucida Console'>)</font>, <font color='#BB00BB'>cpu_grad</font><font face='Lucida Console'>(</font>output_tensor<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>cuda_compute</font><font face='Lucida Console'>(</font>y.<font color='#BB00BB'>begin</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, output_tensor, cuda_grad, cuda_loss<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>cpu_compute</font><font face='Lucida Console'>(</font>y.<font color='#BB00BB'>begin</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, output_tensor, cpu_grad, cpu_loss<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>cuda_grad.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> cpu_grad.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> cuda_grad.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>::std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#5555FF'>*</font><font face='Lucida Console'>(</font>cuda_grad.<font color='#BB00BB'>begin</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> i<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> <font color='#5555FF'>*</font><font face='Lucida Console'>(</font>cpu_grad.<font color='#BB00BB'>begin</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> i<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>8</font><font face='Lucida Console'>)</font>;
        <b>}</b>
        <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> err <font color='#5555FF'>=</font> <font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>cuda_loss <font color='#5555FF'>-</font> cpu_loss<font face='Lucida Console'>)</font> <font color='#5555FF'>/</font> cpu_loss;
        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>err <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font>, "<font color='#CC0000'>binary log per pixel cuda and cpu losses differ</font>"<font face='Lucida Console'>)</font>;
<font color='#0000FF'>#endif</font>
    <b>}</b>

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'><u>void</u></font> <b><a name='test_loss_multiclass_per_pixel_learned_params_on_trivial_single_pixel_task'></a>test_loss_multiclass_per_pixel_learned_params_on_trivial_single_pixel_task</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

        constexpr uint16_t num_classes <font color='#5555FF'>=</font> <font color='#979000'>7</font>;
        constexpr uint16_t true_label <font color='#5555FF'>=</font> num_classes <font color='#5555FF'>/</font> <font color='#979000'>2</font>;

        ::std::vector<font color='#5555FF'>&lt;</font>matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font> <font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><b>{</b> matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font>,<font color='#979000'>1</font>,<font color='#979000'>1</font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font><b>{</b> <font color='#979000'>1</font> <b>}</b><font face='Lucida Console'>)</font> <b>}</b><font face='Lucida Console'>)</font>;
        ::std::vector<font color='#5555FF'>&lt;</font>matrix<font color='#5555FF'>&lt;</font>uint16_t<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font> <font color='#BB00BB'>y</font><font face='Lucida Console'>(</font><b>{</b> matrix<font color='#5555FF'>&lt;</font>uint16_t,<font color='#979000'>1</font>,<font color='#979000'>1</font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font><b>{</b> true_label <b>}</b><font face='Lucida Console'>)</font> <b>}</b><font face='Lucida Console'>)</font>;

        <font color='#0000FF'>using</font> net_type <font color='#5555FF'>=</font> loss_multiclass_log_per_pixel<font color='#5555FF'>&lt;</font>con<font color='#5555FF'>&lt;</font>num_classes,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,input<font color='#5555FF'>&lt;</font>matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>;
        net_type net;

        dnn_trainer<font color='#5555FF'>&lt;</font>net_type<font color='#5555FF'>&gt;</font> <font color='#BB00BB'>trainer</font><font face='Lucida Console'>(</font>net, <font color='#BB00BB'>sgd</font><font face='Lucida Console'>(</font><font color='#979000'>0</font>,<font color='#979000'>0</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        trainer.<font color='#BB00BB'>set_learning_rate</font><font face='Lucida Console'>(</font><font color='#979000'>1e7</font><font face='Lucida Console'>)</font>;
        trainer.<font color='#BB00BB'>set_max_num_epochs</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>;
        trainer.<font color='#BB00BB'>train</font><font face='Lucida Console'>(</font>x, y<font face='Lucida Console'>)</font>;

        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> learned_params <font color='#5555FF'>=</font> layer<font color='#5555FF'>&lt;</font><font color='#979000'>1</font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font>.<font color='#BB00BB'>layer_details</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_layer_params</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> learned_params_data <font color='#5555FF'>=</font> learned_params.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>int</u></font> is_bias <font color='#5555FF'>=</font> <font color='#979000'>0</font>; is_bias <font color='#5555FF'>&lt;</font><font color='#5555FF'>=</font> <font color='#979000'>1</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>is_bias<font face='Lucida Console'>)</font> <b>{</b>
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font>uint16_t k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'>&lt;</font> num_classes; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font> <b>{</b>
                <font color='#0000FF'><u>size_t</u></font> index <font color='#5555FF'>=</font> k <font color='#5555FF'>+</font> is_bias <font color='#5555FF'>*</font> num_classes;
                <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>index <font color='#5555FF'>&lt;</font> learned_params.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>k <font color='#5555FF'>=</font><font color='#5555FF'>=</font> true_label<font face='Lucida Console'>)</font> <b>{</b>
                    <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>learned_params_data[index] <font color='#5555FF'>&gt;</font> <font color='#979000'>1e5</font><font face='Lucida Console'>)</font>;
                <b>}</b>
                <font color='#0000FF'>else</font> <b>{</b>
                    <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>learned_params_data[index] <font color='#5555FF'>&lt;</font> <font color='#5555FF'>-</font><font color='#979000'>1e5</font><font face='Lucida Console'>)</font>;
                <b>}</b>
            <b>}</b>
        <b>}</b>
    <b>}</b>

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'><u>void</u></font> <b><a name='test_loss_multiclass_per_pixel_activations_on_trivial_single_pixel_task'></a>test_loss_multiclass_per_pixel_activations_on_trivial_single_pixel_task</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

        constexpr <font color='#0000FF'><u>int</u></font> input_height <font color='#5555FF'>=</font> <font color='#979000'>35</font>;
        constexpr <font color='#0000FF'><u>int</u></font> input_width <font color='#5555FF'>=</font> <font color='#979000'>27</font>;
        constexpr <font color='#0000FF'><u>int</u></font> output_height <font color='#5555FF'>=</font> input_height;
        constexpr <font color='#0000FF'><u>int</u></font> output_width <font color='#5555FF'>=</font> input_width;
        constexpr <font color='#0000FF'><u>int</u></font> num_samples <font color='#5555FF'>=</font> <font color='#979000'>7</font>;
        constexpr <font color='#0000FF'><u>int</u></font> num_classes <font color='#5555FF'>=</font> <font color='#979000'>5</font>;

        ::std::vector<font color='#5555FF'>&lt;</font>matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font> <font color='#BB00BB'>x</font><font face='Lucida Console'>(</font>num_samples<font face='Lucida Console'>)</font>;
        ::std::vector<font color='#5555FF'>&lt;</font>matrix<font color='#5555FF'>&lt;</font>uint16_t<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font> <font color='#BB00BB'>y</font><font face='Lucida Console'>(</font>num_samples<font face='Lucida Console'>)</font>;

        matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font> <font color='#BB00BB'>xtmp</font><font face='Lucida Console'>(</font>input_height, input_width<font face='Lucida Console'>)</font>;
        matrix<font color='#5555FF'>&lt;</font>uint16_t<font color='#5555FF'>&gt;</font> <font color='#BB00BB'>ytmp</font><font face='Lucida Console'>(</font>output_height, output_width<font face='Lucida Console'>)</font>;

        ::std::default_random_engine <font color='#BB00BB'>generator</font><font face='Lucida Console'>(</font><font color='#979000'>16</font><font face='Lucida Console'>)</font>;
        ::std::bernoulli_distribution <font color='#BB00BB'>coinflip</font><font face='Lucida Console'>(</font><font color='#979000'>0.5</font><font face='Lucida Console'>)</font>;

        <font color='#0000FF'>using</font> filter_type <font color='#5555FF'>=</font> con<font color='#5555FF'>&lt;</font>num_classes,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,input<font color='#5555FF'>&lt;</font>matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>;

        <font color='#009900'>// Define a "truth" filter
</font>        filter_type truth_filter;
        <font color='#BB00BB'>truth_filter</font><font face='Lucida Console'>(</font>xtmp<font face='Lucida Console'>)</font>; <font color='#009900'>// Set up the convolutional layer
</font>
        <font color='#009900'>// Generate training data
</font>        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>int</u></font> ii <font color='#5555FF'>=</font> <font color='#979000'>0</font>; ii <font color='#5555FF'>&lt;</font> num_samples; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>ii<font face='Lucida Console'>)</font> <b>{</b>
            <font color='#009900'>// Generate random inputs x
</font>            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>int</u></font> jj <font color='#5555FF'>=</font> <font color='#979000'>0</font>; jj <font color='#5555FF'>&lt;</font> input_height; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>jj<font face='Lucida Console'>)</font>
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>int</u></font> kk <font color='#5555FF'>=</font> <font color='#979000'>0</font>; kk <font color='#5555FF'>&lt;</font> input_width; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>kk<font face='Lucida Console'>)</font>
                    <font color='#BB00BB'>xtmp</font><font face='Lucida Console'>(</font>jj, kk<font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#BB00BB'>coinflip</font><font face='Lucida Console'>(</font>generator<font face='Lucida Console'>)</font> ? <font color='#979000'>1.f</font> : <font color='#5555FF'>-</font><font color='#979000'>1.f</font>;
            x[ii] <font color='#5555FF'>=</font> xtmp;

            <font color='#009900'>// Generate target output y by applying the truth filter on x
</font>            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> output <font color='#5555FF'>=</font> <font color='#BB00BB'>truth_filter</font><font face='Lucida Console'>(</font>xtmp<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> <font color='#0000FF'>const</font> out_data <font color='#5555FF'>=</font> output.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> out_element <font color='#5555FF'>=</font> [<font color='#5555FF'>&amp;</font>]<font face='Lucida Console'>(</font><font color='#0000FF'><u>int</u></font> row, <font color='#0000FF'><u>int</u></font> column, <font color='#0000FF'><u>int</u></font> k<font face='Lucida Console'>)</font> <b>{</b>
                <font color='#0000FF'>return</font> out_data[<font face='Lucida Console'>(</font>k <font color='#5555FF'>*</font> output.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> row<font face='Lucida Console'>)</font> <font color='#5555FF'>*</font> output.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> column];
            <b>}</b>;

            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>int</u></font> jj <font color='#5555FF'>=</font> <font color='#979000'>0</font>; jj <font color='#5555FF'>&lt;</font> output_height; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>jj<font face='Lucida Console'>)</font> <b>{</b>
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>int</u></font> kk <font color='#5555FF'>=</font> <font color='#979000'>0</font>; kk <font color='#5555FF'>&lt;</font> output_width; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>kk<font face='Lucida Console'>)</font> <b>{</b>
                    uint16_t label <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
                    <font color='#0000FF'><u>float</u></font> max_value <font color='#5555FF'>=</font> <font color='#BB00BB'>out_element</font><font face='Lucida Console'>(</font>jj, kk, <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
                    <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>1</font>; k <font color='#5555FF'>&lt;</font> num_classes; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font> <b>{</b>
                        <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> value <font color='#5555FF'>=</font> <font color='#BB00BB'>out_element</font><font face='Lucida Console'>(</font>jj, kk, k<font face='Lucida Console'>)</font>;
                        <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>value <font color='#5555FF'>&gt;</font> max_value<font face='Lucida Console'>)</font> <b>{</b>
                            label <font color='#5555FF'>=</font> <font color='#0000FF'>static_cast</font><font color='#5555FF'>&lt;</font>uint16_t<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>k<font face='Lucida Console'>)</font>;
                            max_value <font color='#5555FF'>=</font> value;
                        <b>}</b>
                    <b>}</b>
                    <font color='#BB00BB'>ytmp</font><font face='Lucida Console'>(</font>jj, kk<font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> label;
                <b>}</b>
            <b>}</b>
            y[ii] <font color='#5555FF'>=</font> ytmp;
        <b>}</b>

        <font color='#0000FF'>using</font> net_type <font color='#5555FF'>=</font> loss_multiclass_log_per_pixel<font color='#5555FF'>&lt;</font>filter_type<font color='#5555FF'>&gt;</font>;
        net_type net;

        dnn_trainer<font color='#5555FF'>&lt;</font>net_type<font color='#5555FF'>&gt;</font> <font color='#BB00BB'>trainer</font><font face='Lucida Console'>(</font>net, <font color='#BB00BB'>sgd</font><font face='Lucida Console'>(</font><font color='#979000'>0</font>,<font color='#979000'>0</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        trainer.<font color='#BB00BB'>set_learning_rate</font><font face='Lucida Console'>(</font><font color='#979000'>1e6</font><font face='Lucida Console'>)</font>;
        trainer.<font color='#BB00BB'>set_max_num_epochs</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>;
        trainer.<font color='#BB00BB'>train</font><font face='Lucida Console'>(</font>x, y<font face='Lucida Console'>)</font>;

        <font color='#009900'>// Feed forward the training samples.
</font>        resizable_tensor temp_tensor;
        net.<font color='#BB00BB'>to_tensor</font><font face='Lucida Console'>(</font><font color='#5555FF'>&amp;</font>x[<font color='#979000'>0</font>], <font color='#5555FF'>&amp;</font>x[<font color='#979000'>0</font>] <font color='#5555FF'>+</font> num_samples, temp_tensor<font face='Lucida Console'>)</font>;
        net.<font color='#BB00BB'>forward</font><font face='Lucida Console'>(</font>temp_tensor<font face='Lucida Console'>)</font>;
        <font color='#0000FF'>const</font> dimpl::subnet_wrapper<font color='#5555FF'>&lt;</font>filter_type<font color='#5555FF'>&gt;</font> <font color='#BB00BB'>wsub</font><font face='Lucida Console'>(</font>net.<font color='#BB00BB'>subnet</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> output_tensor <font color='#5555FF'>=</font> wsub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> <font color='#0000FF'>const</font> out_data <font color='#5555FF'>=</font> output_tensor.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

        <font color='#009900'>// Let's have a look at the activations before softmax. They should be pretty high
</font>        <font color='#009900'>// (in terms of absolute value), because the learning task is trivial.
</font>        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>int</u></font> ii <font color='#5555FF'>=</font> <font color='#979000'>0</font>; ii <font color='#5555FF'>&lt;</font> num_samples; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>ii<font face='Lucida Console'>)</font> <b>{</b>
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>int</u></font> jj <font color='#5555FF'>=</font> <font color='#979000'>0</font>; jj <font color='#5555FF'>&lt;</font> output_height; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>jj<font face='Lucida Console'>)</font> <b>{</b>
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>int</u></font> kk <font color='#5555FF'>=</font> <font color='#979000'>0</font>; kk <font color='#5555FF'>&lt;</font> output_width; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>kk<font face='Lucida Console'>)</font> <b>{</b>
                    <font color='#0000FF'>const</font> uint16_t true_label <font color='#5555FF'>=</font> y[ii]<font face='Lucida Console'>(</font>jj, kk<font face='Lucida Console'>)</font>;

                    <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'>&lt;</font> num_classes; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font> <b>{</b>
                        <font color='#0000FF'>const</font> <font color='#0000FF'><u>size_t</u></font> index <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font face='Lucida Console'>(</font>ii <font color='#5555FF'>*</font> output_tensor.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> k<font face='Lucida Console'>)</font> <font color='#5555FF'>*</font> output_tensor.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> jj<font face='Lucida Console'>)</font> <font color='#5555FF'>*</font> output_tensor.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> kk;
                        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>index <font color='#5555FF'>&lt;</font> output_tensor.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

                        <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>k <font color='#5555FF'>=</font><font color='#5555FF'>=</font> true_label<font face='Lucida Console'>)</font> <b>{</b>
                            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>out_data[index] <font color='#5555FF'>&gt;</font> <font color='#979000'>1e4</font><font face='Lucida Console'>)</font>;
                        <b>}</b>
                        <font color='#0000FF'>else</font> <b>{</b>
                            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>out_data[index] <font color='#5555FF'>&lt;</font> <font color='#5555FF'>-</font><font color='#979000'>1e4</font><font face='Lucida Console'>)</font>;
                        <b>}</b>
                    <b>}</b>
                <b>}</b>
            <b>}</b>
        <b>}</b>
    <b>}</b>

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'><u>void</u></font> <b><a name='test_loss_multiclass_per_pixel_outputs_on_trivial_task'></a>test_loss_multiclass_per_pixel_outputs_on_trivial_task</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

        constexpr <font color='#0000FF'><u>int</u></font> input_height <font color='#5555FF'>=</font> <font color='#979000'>7</font>;
        constexpr <font color='#0000FF'><u>int</u></font> input_width <font color='#5555FF'>=</font> <font color='#979000'>5</font>;
        constexpr <font color='#0000FF'><u>int</u></font> output_height <font color='#5555FF'>=</font> input_height;
        constexpr <font color='#0000FF'><u>int</u></font> output_width <font color='#5555FF'>=</font> input_width;
        constexpr <font color='#0000FF'><u>int</u></font> num_samples <font color='#5555FF'>=</font> <font color='#979000'>7</font>;
        constexpr <font color='#0000FF'><u>int</u></font> num_classes <font color='#5555FF'>=</font> <font color='#979000'>5</font>;
        constexpr <font color='#0000FF'><u>int</u></font> filter_height <font color='#5555FF'>=</font> <font color='#979000'>3</font>;
        constexpr <font color='#0000FF'><u>int</u></font> filter_width <font color='#5555FF'>=</font> <font color='#979000'>3</font>;

        ::std::vector<font color='#5555FF'>&lt;</font>matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font> <font color='#BB00BB'>x</font><font face='Lucida Console'>(</font>num_samples<font face='Lucida Console'>)</font>;
        ::std::vector<font color='#5555FF'>&lt;</font>matrix<font color='#5555FF'>&lt;</font>uint16_t<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font> <font color='#BB00BB'>y</font><font face='Lucida Console'>(</font>num_samples<font face='Lucida Console'>)</font>;

        matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font> <font color='#BB00BB'>xtmp</font><font face='Lucida Console'>(</font>input_height, input_width<font face='Lucida Console'>)</font>;
        matrix<font color='#5555FF'>&lt;</font>uint16_t<font color='#5555FF'>&gt;</font> <font color='#BB00BB'>ytmp</font><font face='Lucida Console'>(</font>output_height, output_width<font face='Lucida Console'>)</font>;

        ::std::default_random_engine <font color='#BB00BB'>generator</font><font face='Lucida Console'>(</font><font color='#979000'>16</font><font face='Lucida Console'>)</font>;
        ::std::bernoulli_distribution <font color='#BB00BB'>coinflip</font><font face='Lucida Console'>(</font><font color='#979000'>0.5</font><font face='Lucida Console'>)</font>;

        <font color='#0000FF'>using</font> filter_type <font color='#5555FF'>=</font> con<font color='#5555FF'>&lt;</font>num_classes, filter_height, filter_width, <font color='#979000'>1</font>, <font color='#979000'>1</font>, input<font color='#5555FF'>&lt;</font>matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>;

        <font color='#009900'>// Define a "truth" filter
</font>        filter_type truth_filter;
        <font color='#BB00BB'>truth_filter</font><font face='Lucida Console'>(</font>xtmp<font face='Lucida Console'>)</font>; <font color='#009900'>// Set up the convolutional layer
</font>
        <font color='#009900'>// Generate training data
</font>        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>int</u></font> ii <font color='#5555FF'>=</font> <font color='#979000'>0</font>; ii <font color='#5555FF'>&lt;</font> num_samples; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>ii<font face='Lucida Console'>)</font> <b>{</b>
            <font color='#009900'>// Generate random inputs x
</font>            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>int</u></font> jj <font color='#5555FF'>=</font> <font color='#979000'>0</font>; jj <font color='#5555FF'>&lt;</font> input_height; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>jj<font face='Lucida Console'>)</font>
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>int</u></font> kk <font color='#5555FF'>=</font> <font color='#979000'>0</font>; kk <font color='#5555FF'>&lt;</font> input_width; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>kk<font face='Lucida Console'>)</font>
                    <font color='#BB00BB'>xtmp</font><font face='Lucida Console'>(</font>jj, kk<font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#BB00BB'>coinflip</font><font face='Lucida Console'>(</font>generator<font face='Lucida Console'>)</font> ? <font color='#979000'>1.f</font> : <font color='#5555FF'>-</font><font color='#979000'>1.f</font>;
            x[ii] <font color='#5555FF'>=</font> xtmp;

            <font color='#009900'>// Generate target output y by applying the truth filter on x
</font>            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> output <font color='#5555FF'>=</font> <font color='#BB00BB'>truth_filter</font><font face='Lucida Console'>(</font>xtmp<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> <font color='#0000FF'>const</font> out_data <font color='#5555FF'>=</font> output.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> out_element <font color='#5555FF'>=</font> [<font color='#5555FF'>&amp;</font>]<font face='Lucida Console'>(</font><font color='#0000FF'><u>int</u></font> row, <font color='#0000FF'><u>int</u></font> column, <font color='#0000FF'><u>int</u></font> k<font face='Lucida Console'>)</font> <b>{</b>
                <font color='#0000FF'>return</font> out_data[<font face='Lucida Console'>(</font>k <font color='#5555FF'>*</font> output.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> row<font face='Lucida Console'>)</font> <font color='#5555FF'>*</font> output.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> column];
            <b>}</b>;

            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>int</u></font> jj <font color='#5555FF'>=</font> <font color='#979000'>0</font>; jj <font color='#5555FF'>&lt;</font> output_height; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>jj<font face='Lucida Console'>)</font> <b>{</b>
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>int</u></font> kk <font color='#5555FF'>=</font> <font color='#979000'>0</font>; kk <font color='#5555FF'>&lt;</font> output_width; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>kk<font face='Lucida Console'>)</font> <b>{</b>
                    uint16_t label <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
                    <font color='#0000FF'><u>float</u></font> max_value <font color='#5555FF'>=</font> <font color='#BB00BB'>out_element</font><font face='Lucida Console'>(</font>jj, kk, <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
                    <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>1</font>; k <font color='#5555FF'>&lt;</font> num_classes; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font> <b>{</b>
                        <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> value <font color='#5555FF'>=</font> <font color='#BB00BB'>out_element</font><font face='Lucida Console'>(</font>jj, kk, k<font face='Lucida Console'>)</font>;
                        <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>value <font color='#5555FF'>&gt;</font> max_value<font face='Lucida Console'>)</font> <b>{</b>
                            label <font color='#5555FF'>=</font> <font color='#0000FF'>static_cast</font><font color='#5555FF'>&lt;</font>uint16_t<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>k<font face='Lucida Console'>)</font>;
                            max_value <font color='#5555FF'>=</font> value;
                        <b>}</b>
                    <b>}</b>
                    <font color='#BB00BB'>ytmp</font><font face='Lucida Console'>(</font>jj, kk<font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> label;
                <b>}</b>
            <b>}</b>
            y[ii] <font color='#5555FF'>=</font> ytmp;
        <b>}</b>

        <font color='#0000FF'>using</font> net_type <font color='#5555FF'>=</font> loss_multiclass_log_per_pixel<font color='#5555FF'>&lt;</font>filter_type<font color='#5555FF'>&gt;</font>;
        net_type net;

        dnn_trainer<font color='#5555FF'>&lt;</font>net_type<font color='#5555FF'>&gt;</font> <font color='#BB00BB'>trainer</font><font face='Lucida Console'>(</font>net, <font color='#BB00BB'>sgd</font><font face='Lucida Console'>(</font><font color='#979000'>0</font>, <font color='#979000'>0.9</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        trainer.<font color='#BB00BB'>set_learning_rate</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>;
        trainer.<font color='#BB00BB'>set_max_num_epochs</font><font face='Lucida Console'>(</font><font color='#979000'>2000</font><font face='Lucida Console'>)</font>;
        trainer.<font color='#BB00BB'>train</font><font face='Lucida Console'>(</font>x, y<font face='Lucida Console'>)</font>;

        <font color='#009900'>// The learning task is separable, so the net should have no problem
</font>        <font color='#009900'>// getting all the outputs right.
</font>        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>net</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> y<font face='Lucida Console'>)</font>;
    <b>}</b>

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'><u>void</u></font> <b><a name='test_loss_multiclass_per_pixel_with_noise_and_pixels_to_ignore'></a>test_loss_multiclass_per_pixel_with_noise_and_pixels_to_ignore</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#009900'>// "Semantic segmentation" - see https://github.com/davisking/dlib/issues/288
</font>        <font color='#009900'>// Test learning when some pixels are to be ignored, etc.
</font>
        <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

        constexpr <font color='#0000FF'><u>int</u></font> input_height <font color='#5555FF'>=</font> <font color='#979000'>5</font>;
        constexpr <font color='#0000FF'><u>int</u></font> input_width <font color='#5555FF'>=</font> <font color='#979000'>7</font>;
        constexpr <font color='#0000FF'><u>int</u></font> output_height <font color='#5555FF'>=</font> input_height;
        constexpr <font color='#0000FF'><u>int</u></font> output_width <font color='#5555FF'>=</font> input_width;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>int</u></font> num_samples <font color='#5555FF'>=</font> <font color='#979000'>1000</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>int</u></font> num_classes <font color='#5555FF'>=</font> <font color='#979000'>6</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> ignore_probability <font color='#5555FF'>=</font> <font color='#979000'>0.5</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> noise_probability <font color='#5555FF'>=</font> <font color='#979000'>0.05</font>;

        ::std::default_random_engine <font color='#BB00BB'>generator</font><font face='Lucida Console'>(</font><font color='#979000'>16</font><font face='Lucida Console'>)</font>;
        ::std::bernoulli_distribution <font color='#BB00BB'>ignore</font><font face='Lucida Console'>(</font>ignore_probability<font face='Lucida Console'>)</font>;
        ::std::bernoulli_distribution <font color='#BB00BB'>noise_occurrence</font><font face='Lucida Console'>(</font>noise_probability<font face='Lucida Console'>)</font>;
        ::std::uniform_int_distribution<font color='#5555FF'>&lt;</font>uint16_t<font color='#5555FF'>&gt;</font> <font color='#BB00BB'>noisy_label</font><font face='Lucida Console'>(</font><font color='#979000'>0</font>, num_classes <font color='#5555FF'>-</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;

        ::std::vector<font color='#5555FF'>&lt;</font>matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font> <font color='#BB00BB'>x</font><font face='Lucida Console'>(</font>num_samples<font face='Lucida Console'>)</font>;
        ::std::vector<font color='#5555FF'>&lt;</font>matrix<font color='#5555FF'>&lt;</font>uint16_t<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font> <font color='#BB00BB'>y</font><font face='Lucida Console'>(</font>num_samples<font face='Lucida Console'>)</font>;

        ::std::vector<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>int</u></font><font color='#5555FF'>&gt;</font> <font color='#BB00BB'>truth_histogram</font><font face='Lucida Console'>(</font>num_classes<font face='Lucida Console'>)</font>;

        matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font><font color='#5555FF'>&gt;</font> <font color='#BB00BB'>xtmp</font><font face='Lucida Console'>(</font>input_height, input_width<font face='Lucida Console'>)</font>;
        matrix<font color='#5555FF'>&lt;</font>uint16_t<font color='#5555FF'>&gt;</font> <font color='#BB00BB'>ytmp</font><font face='Lucida Console'>(</font>output_height, output_width<font face='Lucida Console'>)</font>;

        <font color='#009900'>// The function to be learned.
</font>        <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> ground_truth <font color='#5555FF'>=</font> [num_classes]<font face='Lucida Console'>(</font><font color='#0000FF'>const</font> matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&amp;</font> x, <font color='#0000FF'><u>int</u></font> row, <font color='#0000FF'><u>int</u></font> column<font face='Lucida Console'>)</font> <b>{</b>
            <font color='#0000FF'><u>double</u></font> sum <font color='#5555FF'>=</font> <font color='#979000'>0.0</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>int</u></font> first_column <font color='#5555FF'>=</font> std::<font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#979000'>0</font>, column <font color='#5555FF'>-</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>int</u></font> last_column <font color='#5555FF'>=</font> std::<font color='#BB00BB'>min</font><font face='Lucida Console'>(</font><font color='#0000FF'>static_cast</font><font color='#5555FF'>&lt;</font><font color='#0000FF'><u>int</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>x.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>, column <font color='#5555FF'>+</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>int</u></font> c <font color='#5555FF'>=</font> first_column; c <font color='#5555FF'>&lt;</font><font color='#5555FF'>=</font> last_column; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>c<font face='Lucida Console'>)</font> <b>{</b>
                sum <font color='#5555FF'>+</font><font color='#5555FF'>=</font> <font color='#BB00BB'>x</font><font face='Lucida Console'>(</font>row, c<font face='Lucida Console'>)</font>;
            <b>}</b>
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>sum <font color='#5555FF'>&lt;</font> num_classes<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>return</font> <font color='#0000FF'>static_cast</font><font color='#5555FF'>&lt;</font>uint16_t<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>sum<font face='Lucida Console'>)</font>;
        <b>}</b>;

        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font> <font color='#0000FF'><u>int</u></font> ii <font color='#5555FF'>=</font> <font color='#979000'>0</font>; ii <font color='#5555FF'>&lt;</font> num_samples; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>ii <font face='Lucida Console'>)</font> <b>{</b>
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font> <font color='#0000FF'><u>int</u></font> jj <font color='#5555FF'>=</font> <font color='#979000'>0</font>; jj <font color='#5555FF'>&lt;</font> input_height; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>jj <font face='Lucida Console'>)</font> <b>{</b>
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font> <font color='#0000FF'><u>int</u></font> kk <font color='#5555FF'>=</font> <font color='#979000'>0</font>; kk <font color='#5555FF'>&lt;</font> input_width; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>kk <font face='Lucida Console'>)</font> <b>{</b>
                    <font color='#009900'>// Generate numbers between 0 and 2.
</font>                    <font color='#0000FF'><u>double</u></font> value <font color='#5555FF'>=</font> <font color='#0000FF'>static_cast</font><font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>ii <font color='#5555FF'>+</font> jj <font color='#5555FF'>+</font> kk<font face='Lucida Console'>)</font> <font color='#5555FF'>/</font> <font color='#979000'>10.0</font>;
                    value <font color='#5555FF'>-</font><font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#0000FF'>static_cast</font><font color='#5555FF'>&lt;</font><font color='#0000FF'><u>int</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>value<font face='Lucida Console'>)</font> <font color='#5555FF'>/</font> <font color='#979000'>2</font><font face='Lucida Console'>)</font> <font color='#5555FF'>*</font> <font color='#979000'>2</font>;
                    <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>value <font color='#5555FF'>&gt;</font><font color='#5555FF'>=</font> <font color='#979000'>0.0</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> value <font color='#5555FF'>&lt;</font> <font color='#979000'>2.0</font><font face='Lucida Console'>)</font>;
                    <font color='#BB00BB'>xtmp</font><font face='Lucida Console'>(</font>jj, kk<font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> value;
                <b>}</b>
            <b>}</b>
            x[ii] <font color='#5555FF'>=</font> xtmp;

            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font> <font color='#0000FF'><u>int</u></font> jj <font color='#5555FF'>=</font> <font color='#979000'>0</font>; jj <font color='#5555FF'>&lt;</font> output_height; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>jj <font face='Lucida Console'>)</font> <b>{</b>
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font> <font color='#0000FF'><u>int</u></font> kk <font color='#5555FF'>=</font> <font color='#979000'>0</font>; kk <font color='#5555FF'>&lt;</font> output_width; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>kk <font face='Lucida Console'>)</font> <b>{</b>
                    uint16_t truth <font color='#5555FF'>=</font> <font color='#BB00BB'>ground_truth</font><font face='Lucida Console'>(</font>x[ii], jj, kk<font face='Lucida Console'>)</font>;
                    <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>truth <font color='#5555FF'>&lt;</font> num_classes<font face='Lucida Console'>)</font>;
                    <font color='#5555FF'>+</font><font color='#5555FF'>+</font>truth_histogram[truth];
                    <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>ignore</font><font face='Lucida Console'>(</font>generator<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <b>{</b>
                        <font color='#BB00BB'>ytmp</font><font face='Lucida Console'>(</font>jj, kk<font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> loss_multiclass_log_per_pixel_::label_to_ignore;
                    <b>}</b>
                    <font color='#0000FF'>else</font> <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>noise_occurrence</font><font face='Lucida Console'>(</font>generator<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <b>{</b>
                        <font color='#BB00BB'>ytmp</font><font face='Lucida Console'>(</font>jj, kk<font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#BB00BB'>noisy_label</font><font face='Lucida Console'>(</font>generator<font face='Lucida Console'>)</font>;
                    <b>}</b>
                    <font color='#0000FF'>else</font> <b>{</b>
                        <font color='#BB00BB'>ytmp</font><font face='Lucida Console'>(</font>jj, kk<font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> truth;
                    <b>}</b>
                <b>}</b>
            <b>}</b>

            y[ii] <font color='#5555FF'>=</font> ytmp;
        <b>}</b>

        <font color='#0000FF'>const</font> <font color='#0000FF'><u>int</u></font> num_total_elements <font color='#5555FF'>=</font> num_samples <font color='#5555FF'>*</font> output_height <font color='#5555FF'>*</font> output_width;

        <b>{</b> <font color='#009900'>// Require a reasonably balanced truth histogram in order to make sure that a trivial classifier is not enough
</font>            <font color='#0000FF'>const</font> <font color='#0000FF'><u>int</u></font> required_min_histogram_value <font color='#5555FF'>=</font> <font color='#0000FF'>static_cast</font><font color='#5555FF'>&lt;</font><font color='#0000FF'><u>int</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>::std::<font color='#BB00BB'>ceil</font><font face='Lucida Console'>(</font>num_total_elements <font color='#5555FF'>/</font> num_classes <font color='#5555FF'>*</font> <font color='#979000'>0.375</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'>auto</font> histogram_value : truth_histogram<font face='Lucida Console'>)</font> <b>{</b>
                <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>histogram_value <font color='#5555FF'>&gt;</font><font color='#5555FF'>=</font> required_min_histogram_value,
                              "<font color='#CC0000'>Histogram value = </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> histogram_value <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>, required = </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> required_min_histogram_value<font face='Lucida Console'>)</font>;
            <b>}</b>
        <b>}</b>

        <font color='#0000FF'>using</font> net_type <font color='#5555FF'>=</font> loss_multiclass_log_per_pixel<font color='#5555FF'>&lt;</font>bn_con<font color='#5555FF'>&lt;</font>con<font color='#5555FF'>&lt;</font>num_classes,<font color='#979000'>1</font>,input_width,<font color='#979000'>1</font>,<font color='#979000'>1</font>,input<font color='#5555FF'>&lt;</font>matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>;
        net_type net;
        sgd <font color='#BB00BB'>defsolver</font><font face='Lucida Console'>(</font><font color='#979000'>0</font>,<font color='#979000'>0.9</font><font face='Lucida Console'>)</font>;
        dnn_trainer<font color='#5555FF'>&lt;</font>net_type<font color='#5555FF'>&gt;</font> <font color='#BB00BB'>trainer</font><font face='Lucida Console'>(</font>net, defsolver<font face='Lucida Console'>)</font>;
        trainer.<font color='#BB00BB'>set_learning_rate</font><font face='Lucida Console'>(</font><font color='#979000'>0.1</font><font face='Lucida Console'>)</font>;
        trainer.<font color='#BB00BB'>set_min_learning_rate</font><font face='Lucida Console'>(</font><font color='#979000'>0.01</font><font face='Lucida Console'>)</font>;
        trainer.<font color='#BB00BB'>set_mini_batch_size</font><font face='Lucida Console'>(</font><font color='#979000'>50</font><font face='Lucida Console'>)</font>;
        trainer.<font color='#BB00BB'>set_max_num_epochs</font><font face='Lucida Console'>(</font><font color='#979000'>170</font><font face='Lucida Console'>)</font>;
        trainer.<font color='#BB00BB'>train</font><font face='Lucida Console'>(</font>x, y<font face='Lucida Console'>)</font>;

        <font color='#0000FF'>const</font> ::std::vector<font color='#5555FF'>&lt;</font>matrix<font color='#5555FF'>&lt;</font>uint16_t<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font> predictions <font color='#5555FF'>=</font> <font color='#BB00BB'>net</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font>;

        <font color='#0000FF'><u>int</u></font> num_correct <font color='#5555FF'>=</font> <font color='#979000'>0</font>;

        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font> <font color='#0000FF'><u>int</u></font> ii <font color='#5555FF'>=</font> <font color='#979000'>0</font>; ii <font color='#5555FF'>&lt;</font> num_samples; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>ii <font face='Lucida Console'>)</font> <b>{</b>
            <font color='#0000FF'>const</font> matrix<font color='#5555FF'>&lt;</font>uint16_t<font color='#5555FF'>&gt;</font><font color='#5555FF'>&amp;</font> prediction <font color='#5555FF'>=</font> predictions[ii];
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>prediction.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> output_height<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>prediction.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> output_width<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font> <font color='#0000FF'><u>int</u></font> jj <font color='#5555FF'>=</font> <font color='#979000'>0</font>; jj <font color='#5555FF'>&lt;</font> output_height; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>jj <font face='Lucida Console'>)</font>
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font> <font color='#0000FF'><u>int</u></font> kk <font color='#5555FF'>=</font> <font color='#979000'>0</font>; kk <font color='#5555FF'>&lt;</font> output_width; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>kk <font face='Lucida Console'>)</font>
                    <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font> <font color='#BB00BB'>prediction</font><font face='Lucida Console'>(</font>jj, kk<font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#BB00BB'>ground_truth</font><font face='Lucida Console'>(</font>x[ii], jj, kk<font face='Lucida Console'>)</font> <font face='Lucida Console'>)</font>
                        <font color='#5555FF'>+</font><font color='#5555FF'>+</font>num_correct;
        <b>}</b>

        <font color='#009900'>// First some sanity checks.
</font>        <font color='#0000FF'>const</font> <font color='#0000FF'><u>int</u></font> num_correct_max <font color='#5555FF'>=</font> num_total_elements;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>num_correct_max <font color='#5555FF'>=</font><font color='#5555FF'>=</font> ::std::<font color='#BB00BB'>accumulate</font><font face='Lucida Console'>(</font>truth_histogram.<font color='#BB00BB'>begin</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, truth_histogram.<font color='#BB00BB'>end</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, <font color='#979000'>0</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>num_correct <font color='#5555FF'>&lt;</font><font color='#5555FF'>=</font> num_correct_max,
                      "<font color='#CC0000'>Number of correctly classified elements = </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> num_correct <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>, max = </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> num_correct_max<font face='Lucida Console'>)</font>;

        <font color='#009900'>// This is the real test, verifying that we have actually learned something.
</font>        <font color='#0000FF'>const</font> <font color='#0000FF'><u>int</u></font> num_correct_required <font color='#5555FF'>=</font> <font color='#0000FF'>static_cast</font><font color='#5555FF'>&lt;</font><font color='#0000FF'><u>int</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>::std::<font color='#BB00BB'>ceil</font><font face='Lucida Console'>(</font><font color='#979000'>0.9</font> <font color='#5555FF'>*</font> num_correct_max<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>num_correct <font color='#5555FF'>&gt;</font><font color='#5555FF'>=</font> num_correct_required,
                      "<font color='#CC0000'>Number of correctly classified elements = </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> num_correct <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>, required = </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> num_correct_required<font face='Lucida Console'>)</font>;

<font color='#0000FF'>#if</font> DLIB_USE_CUDA
        cuda::compute_loss_multiclass_log_per_pixel cuda_compute;
        cpu::compute_loss_multiclass_log_per_pixel cpu_compute;
        <font color='#0000FF'><u>double</u></font> cuda_loss, cpu_loss;
        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> output_tensor <font color='#5555FF'>=</font> net.<font color='#BB00BB'>subnet</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        resizable_tensor <font color='#BB00BB'>cuda_grad</font><font face='Lucida Console'>(</font>output_tensor<font face='Lucida Console'>)</font>, <font color='#BB00BB'>cpu_grad</font><font face='Lucida Console'>(</font>output_tensor<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>cuda_compute</font><font face='Lucida Console'>(</font>y.<font color='#BB00BB'>begin</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, output_tensor, cuda_grad, cuda_loss<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>cpu_compute</font><font face='Lucida Console'>(</font>y.<font color='#BB00BB'>begin</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, output_tensor, cpu_grad, cpu_loss<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>cuda_grad.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> cpu_grad.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> cuda_grad.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>::std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#5555FF'>*</font><font face='Lucida Console'>(</font>cuda_grad.<font color='#BB00BB'>begin</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> i<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> <font color='#5555FF'>*</font><font face='Lucida Console'>(</font>cpu_grad.<font color='#BB00BB'>begin</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> i<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>8</font><font face='Lucida Console'>)</font>;
        <b>}</b>
        <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> err <font color='#5555FF'>=</font> <font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>cuda_loss <font color='#5555FF'>-</font> cpu_loss<font face='Lucida Console'>)</font> <font color='#5555FF'>/</font> cpu_loss;
        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>err <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font>, "<font color='#CC0000'>multiclass log per pixel cuda and cpu losses differ</font>"<font face='Lucida Console'>)</font>;
<font color='#0000FF'>#endif</font>
    <b>}</b>

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'><u>void</u></font> <b><a name='test_loss_multiclass_per_pixel_weighted'></a>test_loss_multiclass_per_pixel_weighted</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#009900'>// Train with pixel-specific weights
</font>
        <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

        constexpr <font color='#0000FF'><u>int</u></font> input_height <font color='#5555FF'>=</font> <font color='#979000'>5</font>;
        constexpr <font color='#0000FF'><u>int</u></font> input_width <font color='#5555FF'>=</font> <font color='#979000'>7</font>;
        constexpr <font color='#0000FF'><u>int</u></font> output_height <font color='#5555FF'>=</font> input_height;
        constexpr <font color='#0000FF'><u>int</u></font> output_width <font color='#5555FF'>=</font> input_width;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>int</u></font> num_samples <font color='#5555FF'>=</font> <font color='#979000'>1000</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>int</u></font> num_classes <font color='#5555FF'>=</font> <font color='#979000'>6</font>;

        ::std::default_random_engine <font color='#BB00BB'>generator</font><font face='Lucida Console'>(</font><font color='#979000'>16</font><font face='Lucida Console'>)</font>;
        ::std::uniform_real_distribution<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font><font color='#5555FF'>&gt;</font> <font color='#BB00BB'>u01</font><font face='Lucida Console'>(</font><font color='#979000'>0.0</font>, <font color='#979000'>1.0</font><font face='Lucida Console'>)</font>;
        ::std::uniform_int_distribution<font color='#5555FF'>&lt;</font>uint16_t<font color='#5555FF'>&gt;</font> <font color='#BB00BB'>noisy_label</font><font face='Lucida Console'>(</font><font color='#979000'>0</font>, num_classes <font color='#5555FF'>-</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;

        ::std::vector<font color='#5555FF'>&lt;</font>matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font> <font color='#BB00BB'>x</font><font face='Lucida Console'>(</font>num_samples<font face='Lucida Console'>)</font>;
        ::std::vector<font color='#5555FF'>&lt;</font>matrix<font color='#5555FF'>&lt;</font>uint16_t<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font> <font color='#BB00BB'>y</font><font face='Lucida Console'>(</font>num_samples<font face='Lucida Console'>)</font>;

        matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font><font color='#5555FF'>&gt;</font> <font color='#BB00BB'>xtmp</font><font face='Lucida Console'>(</font>input_height, input_width<font face='Lucida Console'>)</font>;
        matrix<font color='#5555FF'>&lt;</font>uint16_t<font color='#5555FF'>&gt;</font> <font color='#BB00BB'>ytmp</font><font face='Lucida Console'>(</font>output_height, output_width<font face='Lucida Console'>)</font>;

        <font color='#009900'>// Generate input data
</font>        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>int</u></font> ii <font color='#5555FF'>=</font> <font color='#979000'>0</font>; ii <font color='#5555FF'>&lt;</font> num_samples; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>ii<font face='Lucida Console'>)</font> <b>{</b>
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>int</u></font> jj <font color='#5555FF'>=</font> <font color='#979000'>0</font>; jj <font color='#5555FF'>&lt;</font> input_height; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>jj<font face='Lucida Console'>)</font> <b>{</b>
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>int</u></font> kk <font color='#5555FF'>=</font> <font color='#979000'>0</font>; kk <font color='#5555FF'>&lt;</font> input_width; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>kk<font face='Lucida Console'>)</font> <b>{</b>
                    <font color='#BB00BB'>xtmp</font><font face='Lucida Console'>(</font>jj, kk<font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#BB00BB'>u01</font><font face='Lucida Console'>(</font>generator<font face='Lucida Console'>)</font>;
                    <font color='#BB00BB'>ytmp</font><font face='Lucida Console'>(</font>jj, kk<font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#BB00BB'>noisy_label</font><font face='Lucida Console'>(</font>generator<font face='Lucida Console'>)</font>;
                <b>}</b>
            <b>}</b>
            x[ii] <font color='#5555FF'>=</font> xtmp;
            y[ii] <font color='#5555FF'>=</font> ytmp;
        <b>}</b>

        <font color='#0000FF'>using</font> net_type <font color='#5555FF'>=</font> loss_multiclass_log_per_pixel_weighted<font color='#5555FF'>&lt;</font>con<font color='#5555FF'>&lt;</font>num_classes,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,input<font color='#5555FF'>&lt;</font>matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>;
        <font color='#0000FF'>using</font> weighted_label <font color='#5555FF'>=</font> loss_multiclass_log_per_pixel_weighted_::weighted_label;

        ::std::vector<font color='#5555FF'>&lt;</font>matrix<font color='#5555FF'>&lt;</font>weighted_label<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font> <font color='#BB00BB'>y_weighted</font><font face='Lucida Console'>(</font>num_samples<font face='Lucida Console'>)</font>;

        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>int</u></font> weighted_class <font color='#5555FF'>=</font> <font color='#979000'>0</font>; weighted_class <font color='#5555FF'>&lt;</font> num_classes; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>weighted_class<font face='Lucida Console'>)</font> <b>{</b>

            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

            <font color='#009900'>// Assign weights
</font>            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>int</u></font> ii <font color='#5555FF'>=</font> <font color='#979000'>0</font>; ii <font color='#5555FF'>&lt;</font> num_samples; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>ii<font face='Lucida Console'>)</font> <b>{</b>
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>weighted_class <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font> <b>{</b>
                    y_weighted[ii].<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font>input_height, input_width<font face='Lucida Console'>)</font>;
                <b>}</b>
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>int</u></font> jj <font color='#5555FF'>=</font> <font color='#979000'>0</font>; jj <font color='#5555FF'>&lt;</font> input_height; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>jj<font face='Lucida Console'>)</font> <b>{</b>
                    <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>int</u></font> kk <font color='#5555FF'>=</font> <font color='#979000'>0</font>; kk <font color='#5555FF'>&lt;</font> input_width; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>kk<font face='Lucida Console'>)</font> <b>{</b>
                        <font color='#0000FF'>const</font> uint16_t label <font color='#5555FF'>=</font> y[ii]<font face='Lucida Console'>(</font>jj, kk<font face='Lucida Console'>)</font>;
                        <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> weight
                            <font color='#5555FF'>=</font> label <font color='#5555FF'>=</font><font color='#5555FF'>=</font> weighted_class
                            ? <font color='#979000'>1.1f</font>
                            : <font color='#979000'>0.9f</font>;
                        y_weighted[ii]<font face='Lucida Console'>(</font>jj, kk<font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#BB00BB'>weighted_label</font><font face='Lucida Console'>(</font>label, weight<font face='Lucida Console'>)</font>;
                    <b>}</b>
                <b>}</b>
            <b>}</b>

            net_type net;
            sgd <font color='#BB00BB'>defsolver</font><font face='Lucida Console'>(</font><font color='#979000'>0</font>,<font color='#979000'>0.9</font><font face='Lucida Console'>)</font>;
            dnn_trainer<font color='#5555FF'>&lt;</font>net_type<font color='#5555FF'>&gt;</font> <font color='#BB00BB'>trainer</font><font face='Lucida Console'>(</font>net, defsolver<font face='Lucida Console'>)</font>;
            trainer.<font color='#BB00BB'>set_learning_rate</font><font face='Lucida Console'>(</font><font color='#979000'>0.1</font><font face='Lucida Console'>)</font>;
            trainer.<font color='#BB00BB'>set_min_learning_rate</font><font face='Lucida Console'>(</font><font color='#979000'>0.01</font><font face='Lucida Console'>)</font>;
            trainer.<font color='#BB00BB'>set_mini_batch_size</font><font face='Lucida Console'>(</font><font color='#979000'>10</font><font face='Lucida Console'>)</font>;
            trainer.<font color='#BB00BB'>set_max_num_epochs</font><font face='Lucida Console'>(</font><font color='#979000'>10</font><font face='Lucida Console'>)</font>;
            trainer.<font color='#BB00BB'>train</font><font face='Lucida Console'>(</font>x, y_weighted<font face='Lucida Console'>)</font>;

            <font color='#0000FF'>const</font> ::std::vector<font color='#5555FF'>&lt;</font>matrix<font color='#5555FF'>&lt;</font>uint16_t<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font> predictions <font color='#5555FF'>=</font> <font color='#BB00BB'>net</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font>;

            <font color='#0000FF'><u>int</u></font> num_weighted_class <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            <font color='#0000FF'><u>int</u></font> num_not_weighted_class <font color='#5555FF'>=</font> <font color='#979000'>0</font>;

            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font> <font color='#0000FF'><u>int</u></font> ii <font color='#5555FF'>=</font> <font color='#979000'>0</font>; ii <font color='#5555FF'>&lt;</font> num_samples; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>ii <font face='Lucida Console'>)</font> <b>{</b>
                <font color='#0000FF'>const</font> matrix<font color='#5555FF'>&lt;</font>uint16_t<font color='#5555FF'>&gt;</font><font color='#5555FF'>&amp;</font> prediction <font color='#5555FF'>=</font> predictions[ii];
                <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>prediction.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> output_height<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>prediction.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> output_width<font face='Lucida Console'>)</font>;
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font> <font color='#0000FF'><u>int</u></font> jj <font color='#5555FF'>=</font> <font color='#979000'>0</font>; jj <font color='#5555FF'>&lt;</font> output_height; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>jj <font face='Lucida Console'>)</font>
                    <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font> <font color='#0000FF'><u>int</u></font> kk <font color='#5555FF'>=</font> <font color='#979000'>0</font>; kk <font color='#5555FF'>&lt;</font> output_width; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>kk <font face='Lucida Console'>)</font>
                        <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font> <font color='#BB00BB'>prediction</font><font face='Lucida Console'>(</font>jj, kk<font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> weighted_class <font face='Lucida Console'>)</font>
                            <font color='#5555FF'>+</font><font color='#5555FF'>+</font>num_weighted_class;
                        <font color='#0000FF'>else</font> 
                            <font color='#5555FF'>+</font><font color='#5555FF'>+</font>num_not_weighted_class;
            <b>}</b>

            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>num_weighted_class <font color='#5555FF'>&gt;</font> num_not_weighted_class,
                          "<font color='#CC0000'>The weighted class (</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> weighted_class <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>) does not dominate: </font>"
                          <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> num_weighted_class <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> &lt;= </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> num_not_weighted_class<font face='Lucida Console'>)</font>;

<font color='#0000FF'>#if</font> DLIB_USE_CUDA
            cuda::compute_loss_multiclass_log_per_pixel_weighted cuda_compute;
            cpu::compute_loss_multiclass_log_per_pixel_weighted cpu_compute;
            <font color='#0000FF'><u>double</u></font> cuda_loss, cpu_loss;
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> output_tensor <font color='#5555FF'>=</font> net.<font color='#BB00BB'>subnet</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            resizable_tensor <font color='#BB00BB'>cuda_grad</font><font face='Lucida Console'>(</font>output_tensor<font face='Lucida Console'>)</font>, <font color='#BB00BB'>cpu_grad</font><font face='Lucida Console'>(</font>output_tensor<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>cuda_compute</font><font face='Lucida Console'>(</font>y_weighted.<font color='#BB00BB'>begin</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, output_tensor, cuda_grad, cuda_loss<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>cpu_compute</font><font face='Lucida Console'>(</font>y_weighted.<font color='#BB00BB'>begin</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, output_tensor, cpu_grad, cpu_loss<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>cuda_grad.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> cpu_grad.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> cuda_grad.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>::std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#5555FF'>*</font><font face='Lucida Console'>(</font>cuda_grad.<font color='#BB00BB'>begin</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> i<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> <font color='#5555FF'>*</font><font face='Lucida Console'>(</font>cpu_grad.<font color='#BB00BB'>begin</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> i<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>8</font><font face='Lucida Console'>)</font>;
            <b>}</b>
            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> err <font color='#5555FF'>=</font> <font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>cuda_loss <font color='#5555FF'>-</font> cpu_loss<font face='Lucida Console'>)</font> <font color='#5555FF'>/</font> cpu_loss;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>err <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font>, "<font color='#CC0000'>multi class log per pixel weighted cuda and cpu losses differ</font>"<font face='Lucida Console'>)</font>;
<font color='#0000FF'>#endif</font>
        <b>}</b>
    <b>}</b>

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'><u>void</u></font> <b><a name='test_loss_multiclass_log_weighted'></a>test_loss_multiclass_log_weighted</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
    <b>{</b>

        <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

        constexpr <font color='#0000FF'><u>int</u></font> input_height <font color='#5555FF'>=</font> <font color='#979000'>5</font>;
        constexpr <font color='#0000FF'><u>int</u></font> input_width <font color='#5555FF'>=</font> <font color='#979000'>7</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>size_t</u></font> num_samples <font color='#5555FF'>=</font> <font color='#979000'>1000</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'><u>size_t</u></font> num_classes <font color='#5555FF'>=</font> <font color='#979000'>4</font>;

        ::std::vector<font color='#5555FF'>&lt;</font>matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font> <font color='#BB00BB'>x</font><font face='Lucida Console'>(</font>num_samples<font face='Lucida Console'>)</font>;
        ::std::vector<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font><font color='#5555FF'>&gt;</font> <font color='#BB00BB'>y</font><font face='Lucida Console'>(</font>num_samples<font face='Lucida Console'>)</font>;

        matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font><font color='#5555FF'>&gt;</font> <font color='#BB00BB'>xtmp</font><font face='Lucida Console'>(</font>input_height, input_width<font face='Lucida Console'>)</font>;

        dlib::rand rnd;
        <font color='#009900'>// Generate input data
</font>        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> ii <font color='#5555FF'>=</font> <font color='#979000'>0</font>; ii <font color='#5555FF'>&lt;</font> num_samples; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>ii<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>int</u></font> jj <font color='#5555FF'>=</font> <font color='#979000'>0</font>; jj <font color='#5555FF'>&lt;</font> input_height; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>jj<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>int</u></font> kk <font color='#5555FF'>=</font> <font color='#979000'>0</font>; kk <font color='#5555FF'>&lt;</font> input_width; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>kk<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#BB00BB'>xtmp</font><font face='Lucida Console'>(</font>jj, kk<font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> rnd.<font color='#BB00BB'>get_random_float</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
                <b>}</b>
            <b>}</b>
            x[ii] <font color='#5555FF'>=</font> xtmp;
            y[ii] <font color='#5555FF'>=</font> rnd.<font color='#BB00BB'>get_integer_in_range</font><font face='Lucida Console'>(</font><font color='#979000'>0</font>, num_classes<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>using</font> net_type <font color='#5555FF'>=</font> loss_multiclass_log_weighted<font color='#5555FF'>&lt;</font>fc<font color='#5555FF'>&lt;</font>num_classes, input<font color='#5555FF'>&lt;</font>matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>;

        ::std::vector<font color='#5555FF'>&lt;</font>weighted_label<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font> <font color='#BB00BB'>y_weighted</font><font face='Lucida Console'>(</font>num_samples<font face='Lucida Console'>)</font>;

        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> weighted_class <font color='#5555FF'>=</font> <font color='#979000'>0</font>; weighted_class <font color='#5555FF'>&lt;</font> num_classes; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>weighted_class<font face='Lucida Console'>)</font>
        <b>{</b>

            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

            <font color='#009900'>// Assign weights
</font>            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> ii <font color='#5555FF'>=</font> <font color='#979000'>0</font>; ii <font color='#5555FF'>&lt;</font> num_samples; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>ii<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>const</font> <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> label <font color='#5555FF'>=</font> y[ii];
                <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> weight
                    <font color='#5555FF'>=</font> label <font color='#5555FF'>=</font><font color='#5555FF'>=</font> weighted_class
                    ? <font color='#979000'>1.4f</font>
                    : <font color='#979000'>0.6f</font>;
                y_weighted[ii] <font color='#5555FF'>=</font> weighted_label<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>label, weight<font face='Lucida Console'>)</font>;
            <b>}</b>

            net_type net;
            sgd <font color='#BB00BB'>defsolver</font><font face='Lucida Console'>(</font><font color='#979000'>0</font>, <font color='#979000'>0.9</font><font face='Lucida Console'>)</font>;
            dnn_trainer<font color='#5555FF'>&lt;</font>net_type<font color='#5555FF'>&gt;</font> <font color='#BB00BB'>trainer</font><font face='Lucida Console'>(</font>net, defsolver<font face='Lucida Console'>)</font>;
            trainer.<font color='#BB00BB'>set_learning_rate</font><font face='Lucida Console'>(</font><font color='#979000'>0.1</font><font face='Lucida Console'>)</font>;
            trainer.<font color='#BB00BB'>set_min_learning_rate</font><font face='Lucida Console'>(</font><font color='#979000'>0.01</font><font face='Lucida Console'>)</font>;
            trainer.<font color='#BB00BB'>set_mini_batch_size</font><font face='Lucida Console'>(</font><font color='#979000'>10</font><font face='Lucida Console'>)</font>;
            trainer.<font color='#BB00BB'>set_max_num_epochs</font><font face='Lucida Console'>(</font><font color='#979000'>10</font><font face='Lucida Console'>)</font>;
            trainer.<font color='#BB00BB'>train</font><font face='Lucida Console'>(</font>x, y_weighted<font face='Lucida Console'>)</font>;

            <font color='#0000FF'>const</font> ::std::vector<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font><font color='#5555FF'>&gt;</font> predictions <font color='#5555FF'>=</font> <font color='#BB00BB'>net</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font>;

            <font color='#0000FF'><u>int</u></font> num_weighted_class <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            <font color='#0000FF'><u>int</u></font> num_not_weighted_class <font color='#5555FF'>=</font> <font color='#979000'>0</font>;

            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> ii <font color='#5555FF'>=</font> <font color='#979000'>0</font>; ii <font color='#5555FF'>&lt;</font> num_samples; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>ii<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>predictions[ii] <font color='#5555FF'>=</font><font color='#5555FF'>=</font> weighted_class<font face='Lucida Console'>)</font>
                    <font color='#5555FF'>+</font><font color='#5555FF'>+</font>num_weighted_class;
                <font color='#0000FF'>else</font>
                    <font color='#5555FF'>+</font><font color='#5555FF'>+</font>num_not_weighted_class;
            <b>}</b>

            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>num_weighted_class <font color='#5555FF'>&gt;</font> num_not_weighted_class,
                          "<font color='#CC0000'>The weighted class (</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> weighted_class <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>) does not dominate: </font>"
                          <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> num_weighted_class <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> &lt;= </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> num_not_weighted_class<font face='Lucida Console'>)</font>;
        <b>}</b>
    <b>}</b>

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'><u>void</u></font> <b><a name='test_loss_multibinary_log'></a>test_loss_multibinary_log</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        dlib::rand rnd;

        <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> dims <font color='#5555FF'>=</font> <font color='#979000'>3</font>;
        <font color='#0000FF'>const</font> std::vector<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font> <font color='#BB00BB'>empty_label</font><font face='Lucida Console'>(</font><font color='#979000'>2</font>, <font color='#5555FF'>-</font><font color='#979000'>1.f</font><font face='Lucida Console'>)</font>;
        std::vector<font color='#5555FF'>&lt;</font>matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font>, <font color='#979000'>0</font>, <font color='#979000'>1</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font> samples;
        std::vector<font color='#5555FF'>&lt;</font>std::vector<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font> <font color='#BB00BB'>labels</font><font face='Lucida Console'>(</font><font color='#979000'>128</font>, empty_label<font face='Lucida Console'>)</font>;

        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> labels.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
        <b>{</b>
            matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font>, <font color='#979000'>0</font>, <font color='#979000'>1</font><font color='#5555FF'>&gt;</font> x <font color='#5555FF'>=</font> matrix_cast<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font><font color='#BB00BB'>randm</font><font face='Lucida Console'>(</font>dims, <font color='#979000'>1</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>*</font> rnd.<font color='#BB00BB'>get_double_in_range</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>, <font color='#979000'>9</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> norm <font color='#5555FF'>=</font> <font color='#BB00BB'>sqrt</font><font face='Lucida Console'>(</font><font color='#BB00BB'>sum</font><font face='Lucida Console'>(</font><font color='#BB00BB'>squared</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>norm <font color='#5555FF'>&lt;</font> <font color='#979000'>3</font><font face='Lucida Console'>)</font>
            <b>{</b>
                labels[i][<font color='#979000'>0</font>] <font color='#5555FF'>=</font> <font color='#979000'>1.f</font>;
            <b>}</b>
            <font color='#0000FF'>else</font> <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#979000'>3</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>=</font> norm <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> norm <font color='#5555FF'>&lt;</font> <font color='#979000'>6</font><font face='Lucida Console'>)</font>
            <b>{</b>
                labels[i][<font color='#979000'>0</font>] <font color='#5555FF'>=</font> <font color='#979000'>1.f</font>;
                labels[i][<font color='#979000'>1</font>] <font color='#5555FF'>=</font> <font color='#979000'>1.f</font>;
            <b>}</b>
            <font color='#0000FF'>else</font>
            <b>{</b>
                labels[i][<font color='#979000'>1</font>] <font color='#5555FF'>=</font> <font color='#979000'>1.f</font>;
            <b>}</b>
            samples.<font color='#BB00BB'>push_back</font><font face='Lucida Console'>(</font>std::<font color='#BB00BB'>move</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>using</font> net_type <font color='#5555FF'>=</font> loss_multibinary_log<font color='#5555FF'>&lt;</font>fc<font color='#5555FF'>&lt;</font><font color='#979000'>2</font>, relu<font color='#5555FF'>&lt;</font>bn_fc<font color='#5555FF'>&lt;</font>fc<font color='#5555FF'>&lt;</font><font color='#979000'>10</font>, input<font color='#5555FF'>&lt;</font>matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font>, <font color='#979000'>0</font>, <font color='#979000'>1</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>;
        net_type net;

        <font color='#0000FF'>auto</font> compute_error <font color='#5555FF'>=</font> [<font color='#5555FF'>&amp;</font>net, <font color='#5555FF'>&amp;</font>samples, <font color='#5555FF'>&amp;</font>labels, dims]<font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> preds <font color='#5555FF'>=</font> <font color='#BB00BB'>net</font><font face='Lucida Console'>(</font>samples<font face='Lucida Console'>)</font>;
            <font color='#0000FF'><u>double</u></font> num_wrong <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> labels.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> j <font color='#5555FF'>=</font> <font color='#979000'>0</font>; j <font color='#5555FF'>&lt;</font> labels[i].<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>j<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font face='Lucida Console'>(</font>labels[i][j] <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> preds[i][j] <font color='#5555FF'>&lt;</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font> <font color='#5555FF'>|</font><font color='#5555FF'>|</font>
                        <font face='Lucida Console'>(</font>labels[i][j] <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> preds[i][j] <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                    <b>{</b>
                        <font color='#5555FF'>+</font><font color='#5555FF'>+</font>num_wrong;
                    <b>}</b>
                <b>}</b>
            <b>}</b>
            <font color='#0000FF'>return</font> num_wrong <font color='#5555FF'>/</font> labels.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>/</font> dims;
        <b>}</b>;

        dnn_trainer<font color='#5555FF'>&lt;</font>net_type<font color='#5555FF'>&gt;</font> <font color='#BB00BB'>trainer</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> error_before <font color='#5555FF'>=</font> <font color='#BB00BB'>compute_error</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        trainer.<font color='#BB00BB'>set_learning_rate</font><font face='Lucida Console'>(</font><font color='#979000'>0.1</font><font face='Lucida Console'>)</font>;
        trainer.<font color='#BB00BB'>set_iterations_without_progress_threshold</font><font face='Lucida Console'>(</font><font color='#979000'>10</font><font face='Lucida Console'>)</font>;
        trainer.<font color='#BB00BB'>set_mini_batch_size</font><font face='Lucida Console'>(</font><font color='#979000'>128</font><font face='Lucida Console'>)</font>;
        trainer.<font color='#BB00BB'>set_min_learning_rate</font><font face='Lucida Console'>(</font><font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>3</font><font face='Lucida Console'>)</font>;
        trainer.<font color='#BB00BB'>train</font><font face='Lucida Console'>(</font>samples, labels<font face='Lucida Console'>)</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> error_after <font color='#5555FF'>=</font> <font color='#BB00BB'>compute_error</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

        <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>error_after <font color='#5555FF'>&lt;</font> error_before <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> error_after <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0</font>, "<font color='#CC0000'>multibinary_log error increased after training</font>"<font face='Lucida Console'>)</font>;
    <b>}</b>

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'><u>void</u></font> <b><a name='test_tensor_resize_bilinear'></a>test_tensor_resize_bilinear</b><font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> samps, <font color='#0000FF'><u>long</u></font> k, <font color='#0000FF'><u>long</u></font> nr, <font color='#0000FF'><u>long</u></font> nc,  <font color='#0000FF'><u>long</u></font> onr, <font color='#0000FF'><u>long</u></font> onc<font face='Lucida Console'>)</font>
    <b>{</b>
        resizable_tensor <font color='#BB00BB'>img</font><font face='Lucida Console'>(</font>samps,k,nr,nc<font face='Lucida Console'>)</font>;
        resizable_tensor <font color='#BB00BB'>out</font><font face='Lucida Console'>(</font>samps,k,onr,onc<font face='Lucida Console'>)</font>;
        resizable_tensor <font color='#BB00BB'>out2</font><font face='Lucida Console'>(</font>samps,k,onr,onc<font face='Lucida Console'>)</font>;

        dlib::rand rnd;
        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>int</u></font> iter <font color='#5555FF'>=</font> <font color='#979000'>0</font>; iter <font color='#5555FF'>&lt;</font> <font color='#979000'>10</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>iter<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

            <font color='#0000FF'>const</font> <font color='#0000FF'><u>size_t</u></font> idx <font color='#5555FF'>=</font> rnd.<font color='#BB00BB'>get_random_64bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font>img.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

            img <font color='#5555FF'>=</font> <font color='#979000'>1</font>;
            img.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[idx] <font color='#5555FF'>=</font> <font color='#979000'>2</font>;
            cpu::<font color='#BB00BB'>resize_bilinear</font><font face='Lucida Console'>(</font>out, img<font face='Lucida Console'>)</font>;
<font color='#0000FF'>#ifdef</font> DLIB_USE_CUDA
            cuda::<font color='#BB00BB'>resize_bilinear</font><font face='Lucida Console'>(</font>out2, img<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>out<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>out2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>#endif</font>

            resizable_tensor gradient_input;
            gradient_input.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>out<font face='Lucida Console'>)</font>;
            tt::tensor_rand rnd;
            rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>gradient_input<font face='Lucida Console'>)</font>;

            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> h <font color='#5555FF'>=</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>2</font>;

            img.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[idx] <font color='#5555FF'>=</font> <font color='#979000'>2</font>;
            cpu::<font color='#BB00BB'>resize_bilinear</font><font face='Lucida Console'>(</font>out, img<font face='Lucida Console'>)</font>;
            <font color='#0000FF'><u>float</u></font> f1 <font color='#5555FF'>=</font> <font color='#BB00BB'>dot</font><font face='Lucida Console'>(</font>out, gradient_input<font face='Lucida Console'>)</font>; 

            img.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[idx] <font color='#5555FF'>=</font> <font color='#979000'>2</font><font color='#5555FF'>+</font>h;
            cpu::<font color='#BB00BB'>resize_bilinear</font><font face='Lucida Console'>(</font>out, img<font face='Lucida Console'>)</font>;
            <font color='#0000FF'><u>float</u></font> f2 <font color='#5555FF'>=</font> <font color='#BB00BB'>dot</font><font face='Lucida Console'>(</font>out, gradient_input<font face='Lucida Console'>)</font>; 

            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> numerical_grad <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font>f2<font color='#5555FF'>-</font>f1<font face='Lucida Console'>)</font><font color='#5555FF'>/</font>h;
            dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>numerical grad: </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> numerical_grad;


            resizable_tensor grad, grad2;
            grad.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>img<font face='Lucida Console'>)</font>;
            grad <font color='#5555FF'>=</font> <font color='#979000'>0.1</font>;
            grad2.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>img<font face='Lucida Console'>)</font>;
            grad2 <font color='#5555FF'>=</font> <font color='#979000'>0.1</font>;

            cpu::<font color='#BB00BB'>resize_bilinear_gradient</font><font face='Lucida Console'>(</font>grad2, gradient_input<font face='Lucida Console'>)</font>;
            dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>analytic grad: </font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> grad2.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[idx]<font color='#5555FF'>-</font><font color='#979000'>0.1</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>numerical_grad <font color='#5555FF'>-</font> grad2.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[idx]<font color='#5555FF'>+</font><font color='#979000'>0.1</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>2</font>, std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>numerical_grad <font color='#5555FF'>-</font> grad2.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[idx]<font color='#5555FF'>+</font><font color='#979000'>0.1</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>  numerical_grad: </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> numerical_grad<font face='Lucida Console'>)</font>;

<font color='#0000FF'>#ifdef</font> DLIB_USE_CUDA
            cuda::<font color='#BB00BB'>resize_bilinear_gradient</font><font face='Lucida Console'>(</font>grad, gradient_input<font face='Lucida Console'>)</font>;
            dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>analytic grad: </font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> grad.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[idx]<font color='#5555FF'>-</font><font color='#979000'>0.1</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>numerical_grad <font color='#5555FF'>-</font> grad.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[idx]<font color='#5555FF'>+</font><font color='#979000'>0.1</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>2</font>, std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>numerical_grad <font color='#5555FF'>-</font> grad.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[idx]<font color='#5555FF'>+</font><font color='#979000'>0.1</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>  numerical_grad: </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> numerical_grad<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>grad<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>grad2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>#endif</font>

        <b>}</b>


        <font color='#009900'>// now test with strided/sub-window calls
</font>        alias_tensor <font color='#BB00BB'>aimg</font><font face='Lucida Console'>(</font>samps, k, nr<font color='#5555FF'>-</font><font color='#979000'>2</font>,nc<font color='#5555FF'>-</font><font color='#979000'>2</font><font face='Lucida Console'>)</font>;
        alias_tensor <font color='#BB00BB'>aout</font><font face='Lucida Console'>(</font>samps, k, onr<font color='#5555FF'>-</font><font color='#979000'>2</font>,onc<font color='#5555FF'>-</font><font color='#979000'>2</font><font face='Lucida Console'>)</font>;
        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>int</u></font> iter <font color='#5555FF'>=</font> <font color='#979000'>0</font>; iter <font color='#5555FF'>&lt;</font> <font color='#979000'>10</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>iter<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

            <font color='#0000FF'>const</font> <font color='#0000FF'><u>size_t</u></font> idx <font color='#5555FF'>=</font> rnd.<font color='#BB00BB'>get_random_64bit_number</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>%</font>img.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

            img <font color='#5555FF'>=</font> <font color='#979000'>1</font>;
            img.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[idx] <font color='#5555FF'>=</font> <font color='#979000'>2</font>;
            out <font color='#5555FF'>=</font> <font color='#979000'>9</font>;
            out2 <font color='#5555FF'>=</font> <font color='#979000'>9</font>;
            <font color='#0000FF'>auto</font> wout <font color='#5555FF'>=</font> <font color='#BB00BB'>aout</font><font face='Lucida Console'>(</font>out, out.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font color='#979000'>1</font><font color='#5555FF'>+</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font> wimg <font color='#5555FF'>=</font> <font color='#BB00BB'>aimg</font><font face='Lucida Console'>(</font>img, img.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font color='#979000'>1</font><font color='#5555FF'>+</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            cpu::<font color='#BB00BB'>resize_bilinear</font><font face='Lucida Console'>(</font>wout,out.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,out.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>out.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,  wimg,img.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,img.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>img.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>#ifdef</font> DLIB_USE_CUDA
            <font color='#0000FF'>auto</font> wout2 <font color='#5555FF'>=</font> <font color='#BB00BB'>aout</font><font face='Lucida Console'>(</font>out2, out2.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font color='#979000'>1</font><font color='#5555FF'>+</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            cuda::<font color='#BB00BB'>resize_bilinear</font><font face='Lucida Console'>(</font>wout2,out2.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,out2.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>out2.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,  wimg,img.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,img.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>img.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>out<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>out2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>#endif</font>


            resizable_tensor gradient_input;
            gradient_input.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>out<font face='Lucida Console'>)</font>;
            tt::tensor_rand rnd;
            rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>gradient_input<font face='Lucida Console'>)</font>;

            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> h <font color='#5555FF'>=</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>2</font>;

            img.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[idx] <font color='#5555FF'>=</font> <font color='#979000'>2</font>;
            out <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            wout <font color='#5555FF'>=</font> <font color='#BB00BB'>aout</font><font face='Lucida Console'>(</font>out, out.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font color='#979000'>1</font><font color='#5555FF'>+</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            wimg <font color='#5555FF'>=</font> <font color='#BB00BB'>aimg</font><font face='Lucida Console'>(</font>img, img.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font color='#979000'>1</font><font color='#5555FF'>+</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            cpu::<font color='#BB00BB'>resize_bilinear</font><font face='Lucida Console'>(</font>wout,out.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,out.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>out.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,  wimg,img.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,img.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>img.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'><u>float</u></font> f1 <font color='#5555FF'>=</font> <font color='#BB00BB'>dot</font><font face='Lucida Console'>(</font>out, gradient_input<font face='Lucida Console'>)</font>; 

            img.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[idx] <font color='#5555FF'>=</font> <font color='#979000'>2</font><font color='#5555FF'>+</font>h;
            out <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            cpu::<font color='#BB00BB'>resize_bilinear</font><font face='Lucida Console'>(</font>wout,out.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,out.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>out.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,  wimg,img.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,img.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>img.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'><u>float</u></font> f2 <font color='#5555FF'>=</font> <font color='#BB00BB'>dot</font><font face='Lucida Console'>(</font>out, gradient_input<font face='Lucida Console'>)</font>; 

            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> numerical_grad <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font>f2<font color='#5555FF'>-</font>f1<font face='Lucida Console'>)</font><font color='#5555FF'>/</font>h;
            dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>numerical grad: </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> numerical_grad;


            resizable_tensor grad, grad2;
            grad.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>img<font face='Lucida Console'>)</font>;
            grad <font color='#5555FF'>=</font> <font color='#979000'>0.1</font>;
            grad2.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>img<font face='Lucida Console'>)</font>;
            grad2 <font color='#5555FF'>=</font> <font color='#979000'>0.1</font>;

            <font color='#0000FF'>auto</font> wgrad2 <font color='#5555FF'>=</font> <font color='#BB00BB'>aimg</font><font face='Lucida Console'>(</font>grad2, grad2.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font color='#979000'>1</font><font color='#5555FF'>+</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font> wgradient_input <font color='#5555FF'>=</font> <font color='#BB00BB'>aout</font><font face='Lucida Console'>(</font>gradient_input, gradient_input.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font color='#979000'>1</font><font color='#5555FF'>+</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            cpu::<font color='#BB00BB'>resize_bilinear_gradient</font><font face='Lucida Console'>(</font>wgrad2,grad2.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,grad2.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>grad2.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,  wgradient_input,gradient_input.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,gradient_input.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>gradient_input.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>analytic grad: </font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> grad2.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[idx]<font color='#5555FF'>-</font><font color='#979000'>0.1</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>numerical_grad <font color='#5555FF'>-</font> grad2.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[idx]<font color='#5555FF'>+</font><font color='#979000'>0.1</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>2</font>, std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>numerical_grad <font color='#5555FF'>-</font> grad2.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[idx]<font color='#5555FF'>+</font><font color='#979000'>0.1</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>  numerical_grad: </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> numerical_grad<font face='Lucida Console'>)</font>;

<font color='#0000FF'>#ifdef</font> DLIB_USE_CUDA
            wgrad2 <font color='#5555FF'>=</font> <font color='#BB00BB'>aimg</font><font face='Lucida Console'>(</font>grad, grad.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font color='#979000'>1</font><font color='#5555FF'>+</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            wgradient_input <font color='#5555FF'>=</font> <font color='#BB00BB'>aout</font><font face='Lucida Console'>(</font>gradient_input, gradient_input.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font color='#979000'>1</font><font color='#5555FF'>+</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            cuda::<font color='#BB00BB'>resize_bilinear_gradient</font><font face='Lucida Console'>(</font>wgrad2,grad.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,grad.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>grad.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,  wgradient_input,gradient_input.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,gradient_input.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>gradient_input.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>analytic grad: </font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> grad.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[idx]<font color='#5555FF'>-</font><font color='#979000'>0.1</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font>std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>numerical_grad <font color='#5555FF'>-</font> grad.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[idx]<font color='#5555FF'>+</font><font color='#979000'>0.1</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>2</font>, std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font>numerical_grad <font color='#5555FF'>-</font> grad.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[idx]<font color='#5555FF'>+</font><font color='#979000'>0.1</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>  numerical_grad: </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> numerical_grad<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST_MSG</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>grad<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>grad2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font>, <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>grad<font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>grad2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>#endif</font>


        <b>}</b>
    <b>}</b>


    <font color='#0000FF'><u>void</u></font> <b><a name='test_serialization'></a>test_serialization</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

        <font color='#0000FF'>using</font> net_type <font color='#5555FF'>=</font> loss_mean_squared<font color='#5555FF'>&lt;</font>fc<font color='#5555FF'>&lt;</font><font color='#979000'>1</font>, input<font color='#5555FF'>&lt;</font>matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>;
        net_type net, net2;

        std::ostringstream out;
        <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>net, out<font face='Lucida Console'>)</font>;
        <font color='#0000FF'>const</font> std::string serialized <font color='#5555FF'>=</font> out.<font color='#BB00BB'>str</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        std::istringstream <font color='#BB00BB'>in</font><font face='Lucida Console'>(</font>serialized<font face='Lucida Console'>)</font>;
        dlib::<font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>net2, in<font face='Lucida Console'>)</font>;
        
        std::vector<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>char</u></font><font color='#5555FF'>&gt;</font> buf1;
        dlib::<font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>buf1<font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> net;
        std::vector<font color='#5555FF'>&lt;</font>uint8_t<font color='#5555FF'>&gt;</font> <font color='#BB00BB'>buf2</font><font face='Lucida Console'>(</font>buf1.<font color='#BB00BB'>begin</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, buf1.<font color='#BB00BB'>end</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        dlib::<font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>buf2<font face='Lucida Console'>)</font> <font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font> net2;
    <b>}</b>

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'><u>void</u></font> <b><a name='test_loss_dot'></a>test_loss_dot</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

        std::vector<font color='#5555FF'>&lt;</font>matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font>,<font color='#979000'>0</font>,<font color='#979000'>1</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font> samples;
        std::vector<font color='#5555FF'>&lt;</font>matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font>,<font color='#979000'>0</font>,<font color='#979000'>1</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font> labels;

        <font color='#0000FF'>const</font> matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font> proj <font color='#5555FF'>=</font> matrix_cast<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font><font color='#BB00BB'>randm</font><font face='Lucida Console'>(</font><font color='#979000'>2</font>,<font color='#979000'>3</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>int</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> <font color='#979000'>128</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#009900'>// The task is going to be to learn the matrix proj.  So we make our
</font>            <font color='#009900'>// training data thusly:
</font>            matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font>,<font color='#979000'>0</font>,<font color='#979000'>1</font><font color='#5555FF'>&gt;</font> x <font color='#5555FF'>=</font> matrix_cast<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font><font color='#BB00BB'>randm</font><font face='Lucida Console'>(</font><font color='#979000'>3</font>,<font color='#979000'>1</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font>,<font color='#979000'>0</font>,<font color='#979000'>1</font><font color='#5555FF'>&gt;</font> y <font color='#5555FF'>=</font> <font color='#BB00BB'>normalize</font><font face='Lucida Console'>(</font>proj<font color='#5555FF'>*</font>x<font face='Lucida Console'>)</font>;
            samples.<font color='#BB00BB'>push_back</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font>;
            labels.<font color='#BB00BB'>push_back</font><font face='Lucida Console'>(</font>y<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>using</font> net_type <font color='#5555FF'>=</font> loss_dot<font color='#5555FF'>&lt;</font>
            l2normalize<font color='#5555FF'>&lt;</font>fc_no_bias<font color='#5555FF'>&lt;</font><font color='#979000'>2</font>, 
            input<font color='#5555FF'>&lt;</font>matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font>,<font color='#979000'>0</font>,<font color='#979000'>1</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font> 
            <font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>;

        net_type net;
        dnn_trainer<font color='#5555FF'>&lt;</font>net_type<font color='#5555FF'>&gt;</font> <font color='#BB00BB'>trainer</font><font face='Lucida Console'>(</font>net, <font color='#BB00BB'>sgd</font><font face='Lucida Console'>(</font><font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>4</font>, <font color='#979000'>0.9</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        trainer.<font color='#BB00BB'>set_learning_rate</font><font face='Lucida Console'>(</font><font color='#979000'>0.01</font><font face='Lucida Console'>)</font>;
        trainer.<font color='#BB00BB'>set_min_learning_rate</font><font face='Lucida Console'>(</font><font color='#979000'>0.0000001</font><font face='Lucida Console'>)</font>;
        trainer.<font color='#BB00BB'>set_mini_batch_size</font><font face='Lucida Console'>(</font><font color='#979000'>128</font><font face='Lucida Console'>)</font>;
        trainer.<font color='#BB00BB'>set_max_num_epochs</font><font face='Lucida Console'>(</font><font color='#979000'>50000</font><font face='Lucida Console'>)</font>;
        trainer.<font color='#BB00BB'>train</font><font face='Lucida Console'>(</font>samples, labels<font face='Lucida Console'>)</font>;


        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> samples.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font color='#5555FF'>-</font><font color='#BB00BB'>dot</font><font face='Lucida Console'>(</font><font color='#BB00BB'>net</font><font face='Lucida Console'>(</font>samples[i]<font face='Lucida Console'>)</font>,labels[i]<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>0.001</font><font face='Lucida Console'>)</font>;
        <b>}</b>
    <b>}</b>

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'><u>void</u></font> <b><a name='test_loss_multimulticlass_log'></a>test_loss_multimulticlass_log</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        std::map<font color='#5555FF'>&lt;</font>string,std::vector<font color='#5555FF'>&lt;</font>string<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font> all_labels;
        all_labels["<font color='#CC0000'>c1</font>"] <font color='#5555FF'>=</font> <b>{</b>"<font color='#CC0000'>a</font>", "<font color='#CC0000'>b</font>", "<font color='#CC0000'>c</font>"<b>}</b>;
        all_labels["<font color='#CC0000'>c2</font>"] <font color='#5555FF'>=</font> <b>{</b>"<font color='#CC0000'>d</font>", "<font color='#CC0000'>e</font>", "<font color='#CC0000'>f</font>"<b>}</b>;

        <font color='#009900'>// make training data
</font>        std::vector<font color='#5555FF'>&lt;</font>matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font> samples;
        std::vector<font color='#5555FF'>&lt;</font>std::map<font color='#5555FF'>&lt;</font>string,string<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font> labels;
        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>int</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> <font color='#979000'>3</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>int</u></font> j <font color='#5555FF'>=</font> <font color='#979000'>0</font>; j <font color='#5555FF'>&lt;</font> <font color='#979000'>3</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>j<font face='Lucida Console'>)</font>
            <b>{</b>
                matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font> <font color='#BB00BB'>samp</font><font face='Lucida Console'>(</font><font color='#979000'>2</font>,<font color='#979000'>3</font><font face='Lucida Console'>)</font>;
                samp <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
                <font color='#BB00BB'>samp</font><font face='Lucida Console'>(</font><font color='#979000'>0</font>,i<font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>1</font>;
                <font color='#BB00BB'>samp</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,j<font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>1</font>;
                samples.<font color='#BB00BB'>push_back</font><font face='Lucida Console'>(</font>samp<font face='Lucida Console'>)</font>;

                std::map<font color='#5555FF'>&lt;</font>string,string<font color='#5555FF'>&gt;</font> l;
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>i <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font> l["<font color='#CC0000'>c1</font>"] <font color='#5555FF'>=</font> "<font color='#CC0000'>a</font>";
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>i <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font> l["<font color='#CC0000'>c1</font>"] <font color='#5555FF'>=</font> "<font color='#CC0000'>b</font>";
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>i <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>2</font><font face='Lucida Console'>)</font> l["<font color='#CC0000'>c1</font>"] <font color='#5555FF'>=</font> "<font color='#CC0000'>c</font>";
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>j <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font> l["<font color='#CC0000'>c2</font>"] <font color='#5555FF'>=</font> "<font color='#CC0000'>d</font>";
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>j <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font> l["<font color='#CC0000'>c2</font>"] <font color='#5555FF'>=</font> "<font color='#CC0000'>e</font>";
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>j <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>2</font><font face='Lucida Console'>)</font> l["<font color='#CC0000'>c2</font>"] <font color='#5555FF'>=</font> "<font color='#CC0000'>f</font>";
                labels.<font color='#BB00BB'>push_back</font><font face='Lucida Console'>(</font>l<font face='Lucida Console'>)</font>;
            <b>}</b>
        <b>}</b>

        <font color='#0000FF'>using</font> net_type <font color='#5555FF'>=</font> loss_multimulticlass_log<font color='#5555FF'>&lt;</font>
            fc<font color='#5555FF'>&lt;</font><font color='#979000'>1</font>,        
            input<font color='#5555FF'>&lt;</font>matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font> 
            <font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>;

        net_type <font color='#BB00BB'>net</font><font face='Lucida Console'>(</font>all_labels<font face='Lucida Console'>)</font>;
        net.<font color='#BB00BB'>subnet</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>layer_details</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>set_num_outputs</font><font face='Lucida Console'>(</font>net.<font color='#BB00BB'>loss_details</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>number_of_labels</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

        dnn_trainer<font color='#5555FF'>&lt;</font>net_type<font color='#5555FF'>&gt;</font> <font color='#BB00BB'>trainer</font><font face='Lucida Console'>(</font>net, <font color='#BB00BB'>sgd</font><font face='Lucida Console'>(</font><font color='#979000'>0.1</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        trainer.<font color='#BB00BB'>set_learning_rate</font><font face='Lucida Console'>(</font><font color='#979000'>0.1</font><font face='Lucida Console'>)</font>;
        trainer.<font color='#BB00BB'>set_min_learning_rate</font><font face='Lucida Console'>(</font><font color='#979000'>0.00001</font><font face='Lucida Console'>)</font>;
        trainer.<font color='#BB00BB'>set_iterations_without_progress_threshold</font><font face='Lucida Console'>(</font><font color='#979000'>500</font><font face='Lucida Console'>)</font>;

        trainer.<font color='#BB00BB'>train</font><font face='Lucida Console'>(</font>samples, labels<font face='Lucida Console'>)</font>;

        <font color='#0000FF'>auto</font> predicted_labels <font color='#5555FF'>=</font> <font color='#BB00BB'>net</font><font face='Lucida Console'>(</font>samples<font face='Lucida Console'>)</font>;

        <font color='#009900'>// make sure the network predicts the right labels
</font>        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> samples.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>predicted_labels[i]["<font color='#CC0000'>c1</font>"] <font color='#5555FF'>=</font><font color='#5555FF'>=</font> labels[i]["<font color='#CC0000'>c1</font>"]<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>predicted_labels[i]["<font color='#CC0000'>c2</font>"] <font color='#5555FF'>=</font><font color='#5555FF'>=</font> labels[i]["<font color='#CC0000'>c2</font>"]<font face='Lucida Console'>)</font>;
        <b>}</b>

    <b>}</b>

    <font color='#0000FF'><u>void</u></font> <b><a name='test_layers_scale_and_scale_prev'></a>test_layers_scale_and_scale_prev</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <font color='#0000FF'>using</font> net_type1 <font color='#5555FF'>=</font> scale1<font color='#5555FF'>&lt;</font>con<font color='#5555FF'>&lt;</font><font color='#979000'>3</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,avg_pool_everything<font color='#5555FF'>&lt;</font>tag1<font color='#5555FF'>&lt;</font>input_rgb_image<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>;
        <font color='#0000FF'>using</font> net_type2 <font color='#5555FF'>=</font> scale_prev2<font color='#5555FF'>&lt;</font>skip1<font color='#5555FF'>&lt;</font>tag2<font color='#5555FF'>&lt;</font>con<font color='#5555FF'>&lt;</font><font color='#979000'>3</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,avg_pool_everything<font color='#5555FF'>&lt;</font>tag1<font color='#5555FF'>&lt;</font>input_rgb_image<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>;

        dlib::tt::tensor_rand rnd;
        dlib::resizable_tensor <font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>, <font color='#979000'>3</font>, <font color='#979000'>64</font>, <font color='#979000'>64</font><font face='Lucida Console'>)</font>;
        rnd.<font color='#BB00BB'>fill_gaussian</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font>;
        net_type1 net1;
        net_type2 net2;
        net1.<font color='#BB00BB'>forward</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font>;
        net2.<font color='#BB00BB'>forward</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font>;

        <font color='#009900'>// make sure both convolutional layers have the same weights
</font>        layer<font color='#5555FF'>&lt;</font><font color='#979000'>3</font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net2<font face='Lucida Console'>)</font>.<font color='#BB00BB'>layer_details</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> layer<font color='#5555FF'>&lt;</font><font color='#979000'>1</font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net1<font face='Lucida Console'>)</font>.<font color='#BB00BB'>layer_details</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font> params1 <font color='#5555FF'>=</font> layer<font color='#5555FF'>&lt;</font><font color='#979000'>1</font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net1<font face='Lucida Console'>)</font>.<font color='#BB00BB'>layer_details</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_layer_params</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font> params2 <font color='#5555FF'>=</font> layer<font color='#5555FF'>&lt;</font><font color='#979000'>3</font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net2<font face='Lucida Console'>)</font>.<font color='#BB00BB'>layer_details</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_layer_params</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>params1.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> params2.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> params1.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#5555FF'>*</font><font face='Lucida Console'>(</font>params1.<font color='#BB00BB'>begin</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> i<font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#5555FF'>*</font><font face='Lucida Console'>(</font>params2.<font color='#BB00BB'>begin</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> i<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <b>}</b>
        net2.<font color='#BB00BB'>forward</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font>;

        <font color='#009900'>// make sure both outputs are the same
</font>        <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font> out1 <font color='#5555FF'>=</font> net1.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font> out2 <font color='#5555FF'>=</font> net2.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>out1.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> out2.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> out1.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#5555FF'>*</font><font face='Lucida Console'>(</font>out1.<font color='#BB00BB'>begin</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> i<font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#5555FF'>*</font><font face='Lucida Console'>(</font>out2.<font color='#BB00BB'>begin</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> i<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#009900'>// make sure gradients are the same (within some precision)
</font>        <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> epsilon <font color='#5555FF'>=</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>4</font>;
        dlib::resizable_tensor <font color='#BB00BB'>gradient</font><font face='Lucida Console'>(</font>out1<font face='Lucida Console'>)</font>;
        rnd.<font color='#BB00BB'>fill_gaussian</font><font face='Lucida Console'>(</font>gradient<font face='Lucida Console'>)</font>;

        net1.<font color='#BB00BB'>back_propagate_error</font><font face='Lucida Console'>(</font>x, gradient<font face='Lucida Console'>)</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font> grad1 <font color='#5555FF'>=</font> layer<font color='#5555FF'>&lt;</font><font color='#979000'>1</font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net1<font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_parameter_gradient</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

        net2.<font color='#BB00BB'>back_propagate_error</font><font face='Lucida Console'>(</font>x, gradient<font face='Lucida Console'>)</font>;
        <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font> grad2 <font color='#5555FF'>=</font> layer<font color='#5555FF'>&lt;</font><font color='#979000'>3</font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net2<font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_parameter_gradient</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>grad1.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> grad2.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> grad1.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>::std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#5555FF'>*</font><font face='Lucida Console'>(</font>grad1.<font color='#BB00BB'>begin</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> i<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> <font color='#5555FF'>*</font><font face='Lucida Console'>(</font>grad2.<font color='#BB00BB'>begin</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> i<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> epsilon<font face='Lucida Console'>)</font>;
        <b>}</b>
    <b>}</b>

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'><u>long</u></font> num_filters, <font color='#0000FF'><u>long</u></font> ks, <font color='#0000FF'><u>int</u></font> s, <font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> conp <font color='#5555FF'>=</font> add_layer<font color='#5555FF'>&lt;</font>con_<font color='#5555FF'>&lt;</font>num_filters, ks, ks, s, s, ks<font color='#5555FF'>/</font><font color='#979000'>2</font>, ks<font color='#5555FF'>/</font><font color='#979000'>2</font><font color='#5555FF'>&gt;</font>, SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> INPUT<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> stem <font color='#5555FF'>=</font> add_layer<font color='#5555FF'>&lt;</font>max_pool_<font color='#5555FF'>&lt;</font><font color='#979000'>3</font>, <font color='#979000'>3</font>, <font color='#979000'>2</font>, <font color='#979000'>2</font>, <font color='#979000'>1</font>, <font color='#979000'>1</font><font color='#5555FF'>&gt;</font>, relu<font color='#5555FF'>&lt;</font>bn_con<font color='#5555FF'>&lt;</font>conp<font color='#5555FF'>&lt;</font><font color='#979000'>16</font>, <font color='#979000'>7</font>, <font color='#979000'>2</font>, INPUT<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'><u>long</u></font> num_filters, <font color='#0000FF'><u>long</u></font> growth_rate, <font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> dense_layer <font color='#5555FF'>=</font> concat2<font color='#5555FF'>&lt;</font>tag1, tag2,
                        tag2<font color='#5555FF'>&lt;</font>conp<font color='#5555FF'>&lt;</font>growth_rate, <font color='#979000'>3</font>, <font color='#979000'>1</font>,
                        relu<font color='#5555FF'>&lt;</font>bn_con<font color='#5555FF'>&lt;</font>conp<font color='#5555FF'>&lt;</font><font color='#979000'>4</font> <font color='#5555FF'>*</font> growth_rate, <font color='#979000'>1</font>, <font color='#979000'>1</font>,
                        relu<font color='#5555FF'>&lt;</font>bn_con<font color='#5555FF'>&lt;</font>tag1<font color='#5555FF'>&lt;</font>SUBNET<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> dense_layer_32 <font color='#5555FF'>=</font> dense_layer<font color='#5555FF'>&lt;</font><font color='#979000'>32</font>, <font color='#979000'>8</font>, SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'><u>void</u></font> <b><a name='test_disable_duplicative_biases'></a>test_disable_duplicative_biases</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <font color='#0000FF'>using</font> net_type <font color='#5555FF'>=</font> fc<font color='#5555FF'>&lt;</font><font color='#979000'>10</font>, relu<font color='#5555FF'>&lt;</font>layer_norm<font color='#5555FF'>&lt;</font>fc<font color='#5555FF'>&lt;</font><font color='#979000'>15</font>, relu<font color='#5555FF'>&lt;</font>bn_fc<font color='#5555FF'>&lt;</font>fc<font color='#5555FF'>&lt;</font><font color='#979000'>20</font>,
                         relu<font color='#5555FF'>&lt;</font>layer_norm<font color='#5555FF'>&lt;</font>conp<font color='#5555FF'>&lt;</font><font color='#979000'>32</font>, <font color='#979000'>3</font>, <font color='#979000'>1</font>,
                         repeat<font color='#5555FF'>&lt;</font><font color='#979000'>2</font>, dense_layer_32,
                         stem<font color='#5555FF'>&lt;</font>input_rgb_image<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>;
        net_type net;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>layer<font color='#5555FF'>&lt;</font><font color='#979000'>0</font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font>.<font color='#BB00BB'>layer_details</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>bias_is_disabled</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>false</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>layer<font color='#5555FF'>&lt;</font><font color='#979000'>3</font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font>.<font color='#BB00BB'>layer_details</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>bias_is_disabled</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>false</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>layer<font color='#5555FF'>&lt;</font><font color='#979000'>6</font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font>.<font color='#BB00BB'>layer_details</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>bias_is_disabled</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>false</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>layer<font color='#5555FF'>&lt;</font><font color='#979000'>9</font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font>.<font color='#BB00BB'>layer_details</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>bias_is_disabled</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>false</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>layer<font color='#5555FF'>&lt;</font><font color='#979000'>12</font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font>.<font color='#BB00BB'>layer_details</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>bias_is_disabled</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>false</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>layer<font color='#5555FF'>&lt;</font><font color='#979000'>15</font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font>.<font color='#BB00BB'>layer_details</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>bias_is_disabled</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>false</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>layer<font color='#5555FF'>&lt;</font><font color='#979000'>21</font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font>.<font color='#BB00BB'>layer_details</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>bias_is_disabled</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>false</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>layer<font color='#5555FF'>&lt;</font><font color='#979000'>24</font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font>.<font color='#BB00BB'>layer_details</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>bias_is_disabled</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>false</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>layer<font color='#5555FF'>&lt;</font><font color='#979000'>31</font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font>.<font color='#BB00BB'>layer_details</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>bias_is_disabled</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>false</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>disable_duplicative_biases</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>layer<font color='#5555FF'>&lt;</font><font color='#979000'>0</font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font>.<font color='#BB00BB'>layer_details</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>bias_is_disabled</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>false</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>layer<font color='#5555FF'>&lt;</font><font color='#979000'>3</font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font>.<font color='#BB00BB'>layer_details</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>bias_is_disabled</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>true</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>layer<font color='#5555FF'>&lt;</font><font color='#979000'>6</font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font>.<font color='#BB00BB'>layer_details</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>bias_is_disabled</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>true</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>layer<font color='#5555FF'>&lt;</font><font color='#979000'>9</font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font>.<font color='#BB00BB'>layer_details</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>bias_is_disabled</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>true</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>layer<font color='#5555FF'>&lt;</font><font color='#979000'>12</font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font>.<font color='#BB00BB'>layer_details</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>bias_is_disabled</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>false</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>layer<font color='#5555FF'>&lt;</font><font color='#979000'>15</font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font>.<font color='#BB00BB'>layer_details</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>bias_is_disabled</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>true</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>layer<font color='#5555FF'>&lt;</font><font color='#979000'>21</font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font>.<font color='#BB00BB'>layer_details</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>bias_is_disabled</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>false</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>layer<font color='#5555FF'>&lt;</font><font color='#979000'>24</font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font>.<font color='#BB00BB'>layer_details</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>bias_is_disabled</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>true</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>layer<font color='#5555FF'>&lt;</font><font color='#979000'>31</font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font>.<font color='#BB00BB'>layer_details</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>bias_is_disabled</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>true</font><font face='Lucida Console'>)</font>;
    <b>}</b>

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'><u>void</u></font> <b><a name='test_set_learning_rate_multipliers'></a>test_set_learning_rate_multipliers</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <font color='#0000FF'>using</font> net_type <font color='#5555FF'>=</font> loss_binary_log<font color='#5555FF'>&lt;</font>fc<font color='#5555FF'>&lt;</font><font color='#979000'>2</font>, relu<font color='#5555FF'>&lt;</font>bn_con<font color='#5555FF'>&lt;</font>con<font color='#5555FF'>&lt;</font><font color='#979000'>16</font>, <font color='#979000'>5</font>, <font color='#979000'>5</font>, <font color='#979000'>2</font>, <font color='#979000'>2</font>, input<font color='#5555FF'>&lt;</font>matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>;
        net_type net;
        <font color='#BB00BB'>set_all_learning_rate_multipliers</font><font face='Lucida Console'>(</font>net, <font color='#979000'>0.5</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>layer<font color='#5555FF'>&lt;</font><font color='#979000'>1</font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font>.<font color='#BB00BB'>layer_details</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_learning_rate_multiplier</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0.5</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>layer<font color='#5555FF'>&lt;</font><font color='#979000'>3</font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font>.<font color='#BB00BB'>layer_details</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_learning_rate_multiplier</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0.5</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>layer<font color='#5555FF'>&lt;</font><font color='#979000'>4</font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font>.<font color='#BB00BB'>layer_details</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_learning_rate_multiplier</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0.5</font><font face='Lucida Console'>)</font>;
        set_learning_rate_multipliers_range<font color='#5555FF'>&lt;</font><font color='#979000'>2</font>, <font color='#979000'>4</font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net, <font color='#979000'>0.1</font><font face='Lucida Console'>)</font>;
        set_learning_rate_multipliers_range<font color='#5555FF'>&lt;</font><font color='#979000'>4</font>, <font color='#979000'>6</font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net, <font color='#979000'>0.01</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>layer<font color='#5555FF'>&lt;</font><font color='#979000'>1</font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font>.<font color='#BB00BB'>layer_details</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_learning_rate_multiplier</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0.5</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>layer<font color='#5555FF'>&lt;</font><font color='#979000'>3</font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font>.<font color='#BB00BB'>layer_details</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_learning_rate_multiplier</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0.1</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>layer<font color='#5555FF'>&lt;</font><font color='#979000'>4</font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>net<font face='Lucida Console'>)</font>.<font color='#BB00BB'>layer_details</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_learning_rate_multiplier</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0.01</font><font face='Lucida Console'>)</font>;
    <b>}</b>

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> conblock <font color='#5555FF'>=</font> relu<font color='#5555FF'>&lt;</font>bn_con<font color='#5555FF'>&lt;</font>add_layer<font color='#5555FF'>&lt;</font>con_<font color='#5555FF'>&lt;</font><font color='#979000'>16</font>, <font color='#979000'>3</font>, <font color='#979000'>3</font>, <font color='#979000'>2</font>, <font color='#979000'>2</font>, <font color='#979000'>1</font>, <font color='#979000'>1</font><font color='#5555FF'>&gt;</font>, SUBNET<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>;

    <font color='#0000FF'><u>void</u></font> <b><a name='test_input_ouput_mappers'></a>test_input_ouput_mappers</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#0000FF'>using</font> net_type <font color='#5555FF'>=</font> loss_binary_log_per_pixel<font color='#5555FF'>&lt;</font>con<font color='#5555FF'>&lt;</font><font color='#979000'>1</font>, <font color='#979000'>1</font>, <font color='#979000'>1</font>, <font color='#979000'>1</font>, <font color='#979000'>1</font>,repeat<font color='#5555FF'>&lt;</font><font color='#979000'>3</font>, conblock, tag1<font color='#5555FF'>&lt;</font>input_rgb_image<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>;
        net_type net;
        point <font color='#BB00BB'>p</font><font face='Lucida Console'>(</font><font color='#979000'>32</font>, <font color='#979000'>32</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>input_tensor_to_output_tensor</font><font face='Lucida Console'>(</font>net, p<font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> p <font color='#5555FF'>/</font> <font color='#979000'>8</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>output_tensor_to_input_tensor</font><font face='Lucida Console'>(</font>net, p<font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> p <font color='#5555FF'>*</font> <font color='#979000'>8</font><font face='Lucida Console'>)</font>;
    <b>}</b>

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#009900'>// This test really just checks if the mmod loss goes negative when a whole lot of overlapping
</font>    <font color='#009900'>// truth rectangles are given.  
</font>    <font color='#0000FF'><u>void</u></font> <b><a name='test_loss_mmod'></a>test_loss_mmod</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

        <font color='#009900'>// Define input image size.
</font>        constexpr <font color='#0000FF'><u>int</u></font> nc <font color='#5555FF'>=</font> <font color='#979000'>20</font>;
        constexpr <font color='#0000FF'><u>int</u></font> nr <font color='#5555FF'>=</font> <font color='#979000'>20</font>;

        constexpr <font color='#0000FF'><u>int</u></font> margin <font color='#5555FF'>=</font> <font color='#979000'>3</font>;

        <font color='#009900'>// Create a checkerboard pattern.
</font>        std::deque<font color='#5555FF'>&lt;</font>point<font color='#5555FF'>&gt;</font> labeled_points;
        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>int</u></font> y <font color='#5555FF'>=</font> margin; y <font color='#5555FF'>&lt;</font> nr <font color='#5555FF'>-</font> margin; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>y<font face='Lucida Console'>)</font>
            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>int</u></font> x <font color='#5555FF'>=</font> margin <font color='#5555FF'>+</font> <font color='#979000'>1</font> <font color='#5555FF'>-</font> y <font color='#5555FF'>%</font> <font color='#979000'>2</font>; x <font color='#5555FF'>&lt;</font> nc <font color='#5555FF'>-</font> margin; x <font color='#5555FF'>+</font><font color='#5555FF'>=</font> <font color='#979000'>2</font><font face='Lucida Console'>)</font>
                labeled_points.<font color='#BB00BB'>emplace_back</font><font face='Lucida Console'>(</font>x, y<font face='Lucida Console'>)</font>;

        <font color='#009900'>// Create training data that follows the generated pattern.
</font>        <font color='#0000FF'>typedef</font> matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>&gt;</font> input_image_type;

        <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> generate_input_image <font color='#5555FF'>=</font> [<font color='#5555FF'>&amp;</font>labeled_points, nr, nc]<font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
        <b>{</b>
            input_image_type <font color='#BB00BB'>sample</font><font face='Lucida Console'>(</font>nr, nc<font face='Lucida Console'>)</font>;
            sample <font color='#5555FF'>=</font> <font color='#5555FF'>-</font><font color='#979000'>1.0</font>;

            <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> <font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font> point : labeled_points<font face='Lucida Console'>)</font>
                <font color='#BB00BB'>sample</font><font face='Lucida Console'>(</font>point.<font color='#BB00BB'>y</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, point.<font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>1.0</font>;

            <font color='#0000FF'>return</font> sample;
        <b>}</b>;

        <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> generate_labels <font color='#5555FF'>=</font> [<font color='#5555FF'>&amp;</font>labeled_points]<font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> point_to_rect <font color='#5555FF'>=</font> []<font face='Lucida Console'>(</font><font color='#0000FF'>const</font> point<font color='#5555FF'>&amp;</font> point<font face='Lucida Console'>)</font> <b>{</b>
                constexpr <font color='#0000FF'><u>int</u></font> rect_size <font color='#5555FF'>=</font> <font color='#979000'>5</font>;
                <font color='#0000FF'>return</font> <font color='#BB00BB'>centered_rect</font><font face='Lucida Console'>(</font>
                    point.<font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, point.<font color='#BB00BB'>y</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,
                    rect_size, rect_size
                <font face='Lucida Console'>)</font>;
            <b>}</b>;

            std::vector<font color='#5555FF'>&lt;</font>mmod_rect<font color='#5555FF'>&gt;</font> labels;

            std::<font color='#BB00BB'>transform</font><font face='Lucida Console'>(</font>
                labeled_points.<font color='#BB00BB'>begin</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,
                labeled_points.<font color='#BB00BB'>end</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,
                std::<font color='#BB00BB'>back_inserter</font><font face='Lucida Console'>(</font>labels<font face='Lucida Console'>)</font>,
                point_to_rect
            <font face='Lucida Console'>)</font>;

            <font color='#0000FF'>return</font> labels;
        <b>}</b>;

        <font color='#0000FF'>const</font> input_image_type input_image <font color='#5555FF'>=</font> <font color='#BB00BB'>generate_input_image</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <font color='#0000FF'>const</font> std::vector<font color='#5555FF'>&lt;</font>mmod_rect<font color='#5555FF'>&gt;</font> labels <font color='#5555FF'>=</font> <font color='#BB00BB'>generate_labels</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

        mmod_options <font color='#BB00BB'>options</font><font face='Lucida Console'>(</font>use_image_pyramid::no, <b>{</b> labels <b>}</b><font face='Lucida Console'>)</font>;
        options.be_quiet <font color='#5555FF'>=</font> <font color='#979000'>true</font>;

        <font color='#009900'>// Define a simple network.
</font>        <font color='#0000FF'>using</font> net_type <font color='#5555FF'>=</font> loss_mmod<font color='#5555FF'>&lt;</font>con<font color='#5555FF'>&lt;</font><font color='#979000'>1</font>,<font color='#979000'>5</font>,<font color='#979000'>5</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,con<font color='#5555FF'>&lt;</font><font color='#979000'>1</font>,<font color='#979000'>5</font>,<font color='#979000'>5</font>,<font color='#979000'>2</font>,<font color='#979000'>2</font>,input<font color='#5555FF'>&lt;</font>input_image_type<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>;
        net_type <font color='#BB00BB'>net</font><font face='Lucida Console'>(</font>options<font face='Lucida Console'>)</font>;
        dnn_trainer<font color='#5555FF'>&lt;</font>net_type<font color='#5555FF'>&gt;</font> <font color='#BB00BB'>trainer</font><font face='Lucida Console'>(</font>net, <font color='#BB00BB'>sgd</font><font face='Lucida Console'>(</font><font color='#979000'>0.1</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

        <font color='#009900'>// Train the network. The loss is not supposed to go negative.
</font>        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>int</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'>&lt;</font> <font color='#979000'>100</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font> <b>{</b>
            <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            trainer.<font color='#BB00BB'>train_one_step</font><font face='Lucida Console'>(</font><b>{</b> input_image <b>}</b>, <b>{</b> labels <b>}</b><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>trainer.<font color='#BB00BB'>get_average_loss</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&gt;</font><font color='#5555FF'>=</font> <font color='#979000'>0.0</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#009900'>// Inference should return something for the training data.
</font>        <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> dets <font color='#5555FF'>=</font> <font color='#BB00BB'>net</font><font face='Lucida Console'>(</font>input_image<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>dets.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;

        <font color='#009900'>// Indeed many truth objects should be found.
</font>        <font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> approximate_desired_det_count <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font>nr <font color='#5555FF'>-</font> <font color='#979000'>2</font> <font color='#5555FF'>*</font> margin<font face='Lucida Console'>)</font> <font color='#5555FF'>*</font> <font face='Lucida Console'>(</font>nc <font color='#5555FF'>-</font> <font color='#979000'>2</font> <font color='#5555FF'>*</font> margin<font face='Lucida Console'>)</font> <font color='#5555FF'>/</font> <font color='#979000'>2.0</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>dets.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&gt;</font> approximate_desired_det_count <font color='#5555FF'>*</font> <font color='#979000'>0.45</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font>dets.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> approximate_desired_det_count <font color='#5555FF'>*</font> <font color='#979000'>1.05</font><font face='Lucida Console'>)</font>;
    <b>}</b>

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'><u>void</u></font> <b><a name='test_fuse_layers'></a>test_fuse_layers</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <font color='#0000FF'>using</font> net_type <font color='#5555FF'>=</font> fc<font color='#5555FF'>&lt;</font><font color='#979000'>10</font>, avg_pool_everything<font color='#5555FF'>&lt;</font>relu<font color='#5555FF'>&lt;</font>bn_con<font color='#5555FF'>&lt;</font>con<font color='#5555FF'>&lt;</font><font color='#979000'>16</font>, <font color='#979000'>3</font>, <font color='#979000'>3</font>, <font color='#979000'>1</font>, <font color='#979000'>1</font>, input_rgb_image<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>;
        <font color='#0000FF'>using</font> net_type_fused <font color='#5555FF'>=</font> fc<font color='#5555FF'>&lt;</font><font color='#979000'>10</font>, avg_pool_everything<font color='#5555FF'>&lt;</font>relu<font color='#5555FF'>&lt;</font>affine<font color='#5555FF'>&lt;</font>con<font color='#5555FF'>&lt;</font><font color='#979000'>16</font>, <font color='#979000'>3</font>, <font color='#979000'>3</font>, <font color='#979000'>1</font>, <font color='#979000'>1</font>, input_rgb_image<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>;
        net_type net_bias, net_nobias;
        <font color='#BB00BB'>disable_duplicative_biases</font><font face='Lucida Console'>(</font>net_nobias<font face='Lucida Console'>)</font>;
        resizable_tensor x;
        matrix<font color='#5555FF'>&lt;</font>rgb_pixel<font color='#5555FF'>&gt;</font> <font color='#BB00BB'>image</font><font face='Lucida Console'>(</font><font color='#979000'>8</font>, <font color='#979000'>8</font><font face='Lucida Console'>)</font>;
        net_bias.<font color='#BB00BB'>to_tensor</font><font face='Lucida Console'>(</font><font color='#5555FF'>&amp;</font>image, <font color='#5555FF'>&amp;</font>image<font color='#5555FF'>+</font><font color='#979000'>1</font>, x<font face='Lucida Console'>)</font>;
        net_nobias.<font color='#BB00BB'>to_tensor</font><font face='Lucida Console'>(</font><font color='#5555FF'>&amp;</font>image, <font color='#5555FF'>&amp;</font>image<font color='#5555FF'>+</font><font color='#979000'>1</font>, x<font face='Lucida Console'>)</font>;
        net_bias.<font color='#BB00BB'>forward</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font>;
        net_nobias.<font color='#BB00BB'>forward</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font>;
        net_type_fused <font color='#BB00BB'>net_fused_bias</font><font face='Lucida Console'>(</font>net_bias<font face='Lucida Console'>)</font>;
        net_type_fused <font color='#BB00BB'>net_fused_nobias</font><font face='Lucida Console'>(</font>net_nobias<font face='Lucida Console'>)</font>;
        <font color='#0000FF'>const</font> resizable_tensor out_bias <font color='#5555FF'>=</font> net_bias.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <font color='#0000FF'>const</font> resizable_tensor out_nobias <font color='#5555FF'>=</font> net_nobias.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>fuse_layers</font><font face='Lucida Console'>(</font>net_fused_bias<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>fuse_layers</font><font face='Lucida Console'>(</font>net_fused_nobias<font face='Lucida Console'>)</font>;
        net_fused_bias.<font color='#BB00BB'>forward</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font>;
        net_fused_nobias.<font color='#BB00BB'>forward</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font>;
        <font color='#0000FF'>const</font> resizable_tensor out_bias_fused <font color='#5555FF'>=</font> net_fused_bias.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <font color='#0000FF'>const</font> resizable_tensor out_nobias_fused <font color='#5555FF'>=</font> net_fused_nobias.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>squared</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>out_bias<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>out_bias_fused<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>10</font><font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>squared</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>out_nobias<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>out_nobias_fused<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>10</font><font face='Lucida Console'>)</font>;
    <b>}</b>

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'><u>void</u></font> <b><a name='test_reorg'></a>test_reorg</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
    <b>{</b>
<font color='#0000FF'>#ifdef</font> DLIB_USE_CUDA
        <font color='#BB00BB'>print_spinner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        resizable_tensor <font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font color='#979000'>2</font>, <font color='#979000'>4</font>, <font color='#979000'>8</font>, <font color='#979000'>16</font><font face='Lucida Console'>)</font>;
        resizable_tensor <font color='#BB00BB'>out_cpu</font><font face='Lucida Console'>(</font><font color='#979000'>2</font>, <font color='#979000'>16</font>, <font color='#979000'>4</font>, <font color='#979000'>8</font><font face='Lucida Console'>)</font>, <font color='#BB00BB'>out_cuda</font><font face='Lucida Console'>(</font><font color='#979000'>2</font>, <font color='#979000'>16</font>, <font color='#979000'>4</font>, <font color='#979000'>8</font><font face='Lucida Console'>)</font>;
        resizable_tensor <font color='#BB00BB'>grad_cpu</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font>, <font color='#BB00BB'>grad_cuda</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font>;
        tt::tensor_rand rnd;
        rnd.<font color='#BB00BB'>fill_gaussian</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font>;
        cpu::<font color='#BB00BB'>reorg</font><font face='Lucida Console'>(</font>out_cpu, <font color='#979000'>2</font>, <font color='#979000'>2</font>, x<font face='Lucida Console'>)</font>;
        cuda::<font color='#BB00BB'>reorg</font><font face='Lucida Console'>(</font>out_cuda, <font color='#979000'>2</font>, <font color='#979000'>2</font>, x<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>squared</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>out_cuda<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>out_cpu<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
        cpu::<font color='#BB00BB'>reorg_gradient</font><font face='Lucida Console'>(</font>grad_cpu, <font color='#979000'>2</font>, <font color='#979000'>2</font>, out_cpu<font face='Lucida Console'>)</font>;
        cuda::<font color='#BB00BB'>reorg_gradient</font><font face='Lucida Console'>(</font>grad_cuda, <font color='#979000'>2</font>, <font color='#979000'>2</font>, out_cuda<font face='Lucida Console'>)</font>;
        <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>squared</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>out_cuda<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>out_cpu<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>#endif</font>
    <b>}</b>

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>class</font> <b><a name='dnn_tester'></a>dnn_tester</b> : <font color='#0000FF'>public</font> tester
    <b>{</b>
    <font color='#0000FF'>public</font>:
        <b><a name='dnn_tester'></a>dnn_tester</b> <font face='Lucida Console'>(</font>
        <font face='Lucida Console'>)</font> :
            tester <font face='Lucida Console'>(</font>"<font color='#CC0000'>test_dnn</font>",
                "<font color='#CC0000'>Runs tests on the deep neural network tools.</font>"<font face='Lucida Console'>)</font>
        <b>{</b><b>}</b>

        <font color='#0000FF'><u>void</u></font> <b><a name='run_tests'></a>run_tests</b> <font face='Lucida Console'>(</font>
        <font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#009900'>// make the tests repeatable
</font>            <font color='#BB00BB'>srand</font><font face='Lucida Console'>(</font><font color='#979000'>1234</font><font face='Lucida Console'>)</font>;

            <font color='#BB00BB'>test_tagging</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>#ifdef</font> DLIB_USE_CUDA
            <font color='#BB00BB'>test_affine_rect</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_conv</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_more_ops2</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_more_ops</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,<font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_more_ops</font><font face='Lucida Console'>(</font><font color='#979000'>3</font>,<font color='#979000'>4</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_more_ops</font><font face='Lucida Console'>(</font><font color='#979000'>4</font>,<font color='#979000'>3</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_more_ops</font><font face='Lucida Console'>(</font><font color='#979000'>4</font>,<font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_more_ops</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,<font color='#979000'>4</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_more_ops</font><font face='Lucida Console'>(</font><font color='#979000'>10000</font>,<font color='#979000'>4</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>compare_bn_gpu_and_cpu</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>compare_bn_conv_gpu_and_cpu</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_add</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_multiply_zero_padded</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>compare_adam</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_copy_tensor_gpu</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_copy_tensor_add_to_gpu</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_scale_channels</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>#endif</font>
            <font color='#BB00BB'>test_tensor_resize_bilinear</font><font face='Lucida Console'>(</font><font color='#979000'>2</font>, <font color='#979000'>3</font>, <font color='#979000'>6</font>,<font color='#979000'>6</font>, <font color='#979000'>11</font>, <font color='#979000'>11</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_tensor_resize_bilinear</font><font face='Lucida Console'>(</font><font color='#979000'>2</font>, <font color='#979000'>3</font>, <font color='#979000'>6</font>,<font color='#979000'>6</font>, <font color='#979000'>3</font>, <font color='#979000'>4</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_tensor_resize_bilinear</font><font face='Lucida Console'>(</font><font color='#979000'>2</font>, <font color='#979000'>3</font>, <font color='#979000'>5</font>,<font color='#979000'>6</font>, <font color='#979000'>12</font>, <font color='#979000'>21</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_max_pool</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>2</font>,<font color='#979000'>3</font>,<font color='#979000'>0</font>,<font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_max_pool</font><font face='Lucida Console'>(</font><font color='#979000'>3</font>,<font color='#979000'>3</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>0</font>,<font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_max_pool</font><font face='Lucida Console'>(</font><font color='#979000'>3</font>,<font color='#979000'>3</font>,<font color='#979000'>2</font>,<font color='#979000'>2</font>,<font color='#979000'>0</font>,<font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_max_pool</font><font face='Lucida Console'>(</font><font color='#979000'>2</font>,<font color='#979000'>2</font>,<font color='#979000'>2</font>,<font color='#979000'>2</font>,<font color='#979000'>0</font>,<font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_max_pool</font><font face='Lucida Console'>(</font><font color='#979000'>4</font>,<font color='#979000'>5</font>,<font color='#979000'>3</font>,<font color='#979000'>1</font>,<font color='#979000'>0</font>,<font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_avg_pool</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>2</font>,<font color='#979000'>3</font>,<font color='#979000'>0</font>,<font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_avg_pool</font><font face='Lucida Console'>(</font><font color='#979000'>3</font>,<font color='#979000'>3</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>0</font>,<font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_avg_pool</font><font face='Lucida Console'>(</font><font color='#979000'>3</font>,<font color='#979000'>3</font>,<font color='#979000'>2</font>,<font color='#979000'>2</font>,<font color='#979000'>0</font>,<font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_avg_pool</font><font face='Lucida Console'>(</font><font color='#979000'>2</font>,<font color='#979000'>2</font>,<font color='#979000'>2</font>,<font color='#979000'>2</font>,<font color='#979000'>0</font>,<font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_avg_pool</font><font face='Lucida Console'>(</font><font color='#979000'>4</font>,<font color='#979000'>5</font>,<font color='#979000'>3</font>,<font color='#979000'>1</font>,<font color='#979000'>0</font>,<font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_avg_pool</font><font face='Lucida Console'>(</font><font color='#979000'>4</font>,<font color='#979000'>4</font>,<font color='#979000'>2</font>,<font color='#979000'>2</font>,<font color='#979000'>0</font>,<font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_avg_pool</font><font face='Lucida Console'>(</font><font color='#979000'>4</font>,<font color='#979000'>5</font>,<font color='#979000'>40</font>,<font color='#979000'>50</font>,<font color='#979000'>0</font>,<font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_max_pool</font><font face='Lucida Console'>(</font><font color='#979000'>2</font>,<font color='#979000'>2</font>,<font color='#979000'>2</font>,<font color='#979000'>3</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_max_pool</font><font face='Lucida Console'>(</font><font color='#979000'>3</font>,<font color='#979000'>3</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_max_pool</font><font face='Lucida Console'>(</font><font color='#979000'>3</font>,<font color='#979000'>3</font>,<font color='#979000'>2</font>,<font color='#979000'>2</font>,<font color='#979000'>2</font>,<font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_max_pool</font><font face='Lucida Console'>(</font><font color='#979000'>2</font>,<font color='#979000'>2</font>,<font color='#979000'>2</font>,<font color='#979000'>2</font>,<font color='#979000'>1</font>,<font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_max_pool</font><font face='Lucida Console'>(</font><font color='#979000'>4</font>,<font color='#979000'>5</font>,<font color='#979000'>3</font>,<font color='#979000'>1</font>,<font color='#979000'>2</font>,<font color='#979000'>3</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_avg_pool</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>2</font>,<font color='#979000'>3</font>,<font color='#979000'>0</font>,<font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_avg_pool</font><font face='Lucida Console'>(</font><font color='#979000'>3</font>,<font color='#979000'>3</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font>,<font color='#979000'>2</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_avg_pool</font><font face='Lucida Console'>(</font><font color='#979000'>3</font>,<font color='#979000'>3</font>,<font color='#979000'>2</font>,<font color='#979000'>2</font>,<font color='#979000'>2</font>,<font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_avg_pool</font><font face='Lucida Console'>(</font><font color='#979000'>2</font>,<font color='#979000'>2</font>,<font color='#979000'>2</font>,<font color='#979000'>2</font>,<font color='#979000'>1</font>,<font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_avg_pool</font><font face='Lucida Console'>(</font><font color='#979000'>4</font>,<font color='#979000'>5</font>,<font color='#979000'>3</font>,<font color='#979000'>1</font>,<font color='#979000'>2</font>,<font color='#979000'>4</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_avg_pool</font><font face='Lucida Console'>(</font><font color='#979000'>4</font>,<font color='#979000'>4</font>,<font color='#979000'>2</font>,<font color='#979000'>2</font>,<font color='#979000'>1</font>,<font color='#979000'>3</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_avg_pool</font><font face='Lucida Console'>(</font><font color='#979000'>4</font>,<font color='#979000'>5</font>,<font color='#979000'>40</font>,<font color='#979000'>50</font>,<font color='#979000'>0</font>,<font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_tanh</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_softmax</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_softmax_all</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_sigmoid</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_mish</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_leaky_relu</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_clipped_relu</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_elu</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_gelu</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_smelu</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_silu</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_batch_normalize</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_batch_normalize_conv</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_layer_normalize</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_basic_tensor_ops</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_layers</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_visit_functions</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_copy_tensor_cpu</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_copy_tensor_add_to_cpu</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_concat</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_simple_linear_regression</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_simple_linear_regression_eil</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_simple_linear_regression_with_mult_prev</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_multioutput_linear_regression</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_simple_autoencoder</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_loss_mean_squared_per_channel_and_pixel</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_loss_binary_log_per_pixel_learned_params_on_trivial_two_pixel_task</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_loss_binary_log_per_pixel_outputs_on_trivial_task</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_loss_binary_log_per_pixel_with_noise_and_pixels_to_ignore</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_loss_multiclass_per_pixel_learned_params_on_trivial_single_pixel_task</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_loss_multiclass_per_pixel_activations_on_trivial_single_pixel_task</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_loss_multiclass_per_pixel_outputs_on_trivial_task</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_loss_multiclass_per_pixel_with_noise_and_pixels_to_ignore</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_loss_multiclass_per_pixel_weighted</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_loss_multiclass_log_weighted</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_loss_multibinary_log</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_serialization</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_loss_dot</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_loss_multimulticlass_log</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_loss_mmod</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_layers_scale_and_scale_prev</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_disable_duplicative_biases</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_set_learning_rate_multipliers</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_input_ouput_mappers</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_fuse_layers</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>test_reorg</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'><u>void</u></font> <b><a name='perform_test'></a>perform_test</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
        <b>{</b>
            dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>NOW RUNNING TESTS WITH set_dnn_prefer_fastest_algorithms()</font>";
            <font color='#BB00BB'>set_dnn_prefer_fastest_algorithms</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>run_tests</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

            dlog <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> LINFO <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>NOW RUNNING TESTS WITH set_dnn_prefer_smallest_algorithms()</font>";
            <font color='#BB00BB'>set_dnn_prefer_smallest_algorithms</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>run_tests</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;


            <b>{</b>
                resizable_tensor <font color='#BB00BB'>a</font><font face='Lucida Console'>(</font><font color='#979000'>2</font>,<font color='#979000'>3</font>,<font color='#979000'>4</font>,<font color='#979000'>5</font><font face='Lucida Console'>)</font>;
                resizable_tensor <font color='#BB00BB'>b</font><font face='Lucida Console'>(</font><font color='#979000'>2</font>,<font color='#979000'>3</font>,<font color='#979000'>4</font>,<font color='#979000'>5</font><font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>a,b<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

                a.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font><font color='#979000'>2</font>,<font color='#979000'>3</font>,<font color='#979000'>4</font>,<font color='#979000'>4</font><font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#5555FF'>!</font><font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>a,b<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                a.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font><font color='#979000'>2</font>,<font color='#979000'>3</font>,<font color='#979000'>3</font>,<font color='#979000'>5</font><font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#5555FF'>!</font><font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>a,b<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                a.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font><font color='#979000'>2</font>,<font color='#979000'>2</font>,<font color='#979000'>4</font>,<font color='#979000'>5</font><font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#5555FF'>!</font><font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>a,b<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                a.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,<font color='#979000'>3</font>,<font color='#979000'>4</font>,<font color='#979000'>5</font><font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>DLIB_TEST</font><font face='Lucida Console'>(</font><font color='#5555FF'>!</font><font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>a,b<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

                <font color='#BB00BB'>static_assert</font><font face='Lucida Console'>(</font><font color='#5555FF'>!</font>is_image_type<font color='#5555FF'>&lt;</font>resizable_tensor<font color='#5555FF'>&gt;</font>::value, "<font color='#CC0000'>should be false</font>"<font face='Lucida Console'>)</font>;
            <b>}</b>
        <b>}</b>
    <b>}</b> a;
<b>}</b>

<font color='#0000FF'>#endif</font> <font color='#009900'>// __INTELLISENSE__
</font>

</pre></body></html>